{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "favorite-magnet",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:08.217555Z",
     "iopub.status.busy": "2021-04-25T10:51:08.216667Z",
     "iopub.status.idle": "2021-04-25T10:51:08.221326Z",
     "shell.execute_reply": "2021-04-25T10:51:08.220639Z"
    },
    "id": "IU3QDXWZj56H",
    "papermill": {
     "duration": 0.072901,
     "end_time": "2021-04-25T10:51:08.221502",
     "exception": false,
     "start_time": "2021-04-25T10:51:08.148601",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "TEST = False\n",
    "threshold =  0.47231523177731743\n",
    "\n",
    "GPU = True\n",
    "\n",
    "#The WHERE_PLACE variable must have the following value:\"local\",\"kaggle\",\"colab\"\n",
    "WHERE_PLACE = \"kaggle\" \n",
    "Phase_1 = False\n",
    "Phase_2 = False\n",
    "Phase_3 = False\n",
    "Phase_4 = False\n",
    "NAME_OUTPUT_FILE = 'A_XGBM-tuning(v49)'\n",
    "NAME_OUTPUT_FOR_STAKING = 'XGBM_probs_for_staking(v49).npy'\n",
    "NAME_OUTPUT_FOR_STAKING_TEST = 'XGBM_probs-test_for_staking(v49).npy'\n",
    "TARGET = 'Survived'\n",
    "INDEX_COL = 'PassengerId'\n",
    "\n",
    "\n",
    "# Range of values for selection in phase 1 :\n",
    "import numpy as np\n",
    "N_ESTIMATORS_RANG = list(range(5000,15001,100))#  [50000] #[5000] #[500] #\n",
    "RANG_LEARNING_RATE =   list(np.arange(0.00015,0.0015,0.00005)) # 0.0015 #list(np.arange(5e-3,1e-1,5e-3)) # [0.005]\n",
    "\n",
    "CV =  5 # \n",
    "CV_O =  5 # 10 # \n",
    "TEST_SIZE = 0.2\n",
    "\n",
    "\n",
    "N_TRIALS = 50\n",
    "\n",
    "SEED = 2599520\n",
    "\n",
    "\n",
    "SELECT_PRED = False\n",
    "SELECT_PRED2 = False\n",
    "\n",
    "HIGH_PARAM_END = False\n",
    "\n",
    "# ------------------------------------------------------------------------------\n",
    "# Path definition\n",
    "# ------------------------------------------------------------------------------\n",
    "from pathlib import Path\n",
    "DATA_PATH_Kaggle = Path(\"../input/tabular-playground-series-apr-2021\")\n",
    "DATA_PATH_Colab = Path(\"/content/drive/My Drive/Colab Notebooks/TB2\")\n",
    "LOG_PATH = Path(\"./log/\")\n",
    "NAME_RUN_LOG = 'Run__Logs.log'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "prostate-scanner",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:08.333999Z",
     "iopub.status.busy": "2021-04-25T10:51:08.332970Z",
     "iopub.status.idle": "2021-04-25T10:51:08.339438Z",
     "shell.execute_reply": "2021-04-25T10:51:08.338877Z"
    },
    "papermill": {
     "duration": 0.064472,
     "end_time": "2021-04-25T10:51:08.339593",
     "exception": false,
     "start_time": "2021-04-25T10:51:08.275121",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "27"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(RANG_LEARNING_RATE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "sharing-stupid",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:08.450179Z",
     "iopub.status.busy": "2021-04-25T10:51:08.449432Z",
     "iopub.status.idle": "2021-04-25T10:51:08.452285Z",
     "shell.execute_reply": "2021-04-25T10:51:08.452818Z"
    },
    "papermill": {
     "duration": 0.060715,
     "end_time": "2021-04-25T10:51:08.453016",
     "exception": false,
     "start_time": "2021-04-25T10:51:08.392301",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "####### Берем полученные наилучшие значения гиперпараметров с прошлых итераций\n",
    "params = {\n",
    "     'n_estimators': 20000, #5000,\n",
    "     'learning_rate': 0.0005, # 0.0015,  #0.015, #0.005,\n",
    "    'random_state': SEED,\n",
    "'max_depth': 6, \n",
    "'min_child_weight': 24, \n",
    "'colsample_bytree': 0.6248628797707747, \n",
    "'subsample': 0.7109465438773837, \n",
    "'lambda': 0.6290623246094361, \n",
    "'alpha': 0.32470257316374473,\n",
    "'gamma': 0.9123913042030198\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "built-mobility",
   "metadata": {
    "papermill": {
     "duration": 0.050967,
     "end_time": "2021-04-25T10:51:08.554621",
     "exception": false,
     "start_time": "2021-04-25T10:51:08.503654",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "####### Берем полученные наилучшие значения гиперпараметров с прошлых итераций\n",
    "params = {\n",
    "     'n_estimators': 5000,\n",
    "     'learning_rate': 0.0015,  #0.015, #0.005,\n",
    "    'random_state': SEED,\n",
    "    \n",
    "    'mmax_depth': 6, \n",
    "    'min_child_weight': 88,\n",
    "    \n",
    "'colsample_bytree': 0.7557774256291229, \n",
    "'subsample': 0.6039789065360579, \n",
    "'lambda': 0.40274186379196697, \n",
    "'alpha': 0.6376862488529348, \n",
    "'gamma': 0.7313916309633068\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "numeric-overall",
   "metadata": {
    "papermill": {
     "duration": 0.050699,
     "end_time": "2021-04-25T10:51:08.656183",
     "exception": false,
     "start_time": "2021-04-25T10:51:08.605484",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "####### Берем полученные наилучшие значения гиперпараметров с прошлых итераций\n",
    "params = {\n",
    "     'n_estimators': 5000,\n",
    "     'learning_rate': 0.0015,  #0.015, #0.005,\n",
    "    'random_state': SEED,\n",
    "    \n",
    "    'mmax_depth': 6, \n",
    "    'min_child_weight': 88,\n",
    "    \n",
    "#'colsample_bytree': 0.7557774256291229, \n",
    "#'subsample': 0.6039789065360579, \n",
    "   'colsample_bytree': 0.768634843058465, \n",
    "   'subsample': 0.8738982646237107, \n",
    "    \n",
    "'lambda': 0.40274186379196697, \n",
    "'alpha': 0.6376862488529348, \n",
    "'gamma': 0.7313916309633068\n",
    "}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "subjective-intellectual",
   "metadata": {
    "papermill": {
     "duration": 0.051856,
     "end_time": "2021-04-25T10:51:08.759991",
     "exception": false,
     "start_time": "2021-04-25T10:51:08.708135",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "####### Берем полученные наилучшие значения гиперпараметров с прошлых итераций\n",
    "params = {\n",
    "     'n_estimators': 20000, #5000,\n",
    "     'learning_rate': 0.0009,  #0.0015, #0.005,\n",
    "    \n",
    "    \n",
    "    \n",
    "    'random_state': SEED,\n",
    "    \n",
    "    'mmax_depth': 6, \n",
    "    'min_child_weight': 88,\n",
    "    \n",
    "'colsample_bytree': 0.7557774256291229, \n",
    "'subsample': 0.6039789065360579, \n",
    "'lambda': 0.40274186379196697, \n",
    "'alpha': 0.6376862488529348, \n",
    "'gamma': 0.7313916309633068\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "vulnerable-newsletter",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:08.887787Z",
     "iopub.status.busy": "2021-04-25T10:51:08.877511Z",
     "iopub.status.idle": "2021-04-25T10:51:08.903876Z",
     "shell.execute_reply": "2021-04-25T10:51:08.904465Z"
    },
    "id": "otxlKdSTj56b",
    "outputId": "aa5ed5fe-410d-4872-bdd8-9197c83d5a2a",
    "papermill": {
     "duration": 0.092921,
     "end_time": "2021-04-25T10:51:08.904672",
     "exception": false,
     "start_time": "2021-04-25T10:51:08.811751",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KAGGLE\n",
      "Время старта скрипта: 25-04-2021 10:51\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'n_estimators': 20000,\n",
       " 'learning_rate': 0.0005,\n",
       " 'random_state': 2599520,\n",
       " 'max_depth': 6,\n",
       " 'min_child_weight': 24,\n",
       " 'colsample_bytree': 0.6248628797707747,\n",
       " 'subsample': 0.7109465438773837,\n",
       " 'lambda': 0.6290623246094361,\n",
       " 'alpha': 0.32470257316374473,\n",
       " 'gamma': 0.9123913042030198,\n",
       " 'device': 'gpu',\n",
       " 'gpu_platform_id': 0,\n",
       " 'gpu_device_id': 0,\n",
       " 'tree_method': 'gpu_hist',\n",
       " 'predictor': 'gpu_predictor'}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from time import time\n",
    "import datetime\n",
    "start_time = time()\n",
    "\n",
    "if TEST:\n",
    "    params.update({'n_estimators': 5})\n",
    "    NAME_OUTPUT_FILE = '_TEST_' + NAME_OUTPUT_FILE + '_TEST_'\n",
    "    print('**************************************************************')\n",
    "    print('*** TEST mode ***')\n",
    "    print('**************************************************************')\n",
    "    \n",
    "if WHERE_PLACE == \"local\":\n",
    "    input_path = Path('Data')\n",
    "    d_ext = 'csv'\n",
    "    print(\"LOCAL\")\n",
    "elif  WHERE_PLACE == \"kaggle\":\n",
    "    input_path = DATA_PATH_Kaggle\n",
    "    d_ext = 'csv'\n",
    "    print(\"KAGGLE\")\n",
    "elif  WHERE_PLACE == \"colab\":\n",
    "    from google.colab import drive\n",
    "    drive.mount('/content/drive/')\n",
    "    #!ls -al \"/content/drive/My Drive/Colab Notebooks/TB1\"\n",
    "    input_path = DATA_PATH_Colab\n",
    "    d_ext = 'dcv'\n",
    "    !pip install catboost\n",
    "    !pip install optuna\n",
    "    !pip install category_encoders\n",
    "    print(\"Google Colab\")\n",
    "else:\n",
    "    raise Exception('the WHERE_PLACE variable must have the following value: \"local, kaggle, colab\".')\n",
    "    \n",
    "if GPU:\n",
    "    params['device'] = 'gpu'\n",
    "    params['gpu_platform_id'] = 0\n",
    "    params['gpu_device_id'] = 0\n",
    "    params['tree_method'] = 'gpu_hist'\n",
    "    params['predictor'] = 'gpu_predictor'\n",
    "    \n",
    "LOG_PATH.mkdir(parents=True, exist_ok=True)\n",
    "with open(NAME_RUN_LOG, 'at', encoding='utf-8') as logfile:\n",
    "    print('\\n\\n***********************\\n***********************', \n",
    "          file=logfile)\n",
    "    print('Время старта скрипта: {}'.format(datetime.datetime.now().strftime(\"%d-%m-%Y %H:%M\")), \n",
    "          file=logfile)\n",
    "print('Время старта скрипта: {}'.format(datetime.datetime.now().strftime(\"%d-%m-%Y %H:%M\")))\n",
    "    \n",
    "params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "educational-border",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:09.018424Z",
     "iopub.status.busy": "2021-04-25T10:51:09.017718Z",
     "iopub.status.idle": "2021-04-25T10:51:28.067037Z",
     "shell.execute_reply": "2021-04-25T10:51:28.066269Z"
    },
    "papermill": {
     "duration": 19.107813,
     "end_time": "2021-04-25T10:51:28.067283",
     "exception": false,
     "start_time": "2021-04-25T10:51:08.959470",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting neptune-client\r\n",
      "  Downloading neptune-client-0.9.5.tar.gz (211 kB)\r\n",
      "\u001b[K     |████████████████████████████████| 211 kB 68 kB/s \r\n",
      "\u001b[?25hCollecting bravado\r\n",
      "  Downloading bravado-11.0.3-py2.py3-none-any.whl (38 kB)\r\n",
      "Requirement already satisfied: click>=7.0 in /opt/conda/lib/python3.7/site-packages (from neptune-client) (7.1.2)\r\n",
      "Requirement already satisfied: future>=0.17.1 in /opt/conda/lib/python3.7/site-packages (from neptune-client) (0.18.2)\r\n",
      "Requirement already satisfied: oauthlib>=2.1.0 in /opt/conda/lib/python3.7/site-packages (from neptune-client) (3.0.1)\r\n",
      "Requirement already satisfied: pandas in /opt/conda/lib/python3.7/site-packages (from neptune-client) (1.1.5)\r\n",
      "Requirement already satisfied: Pillow>=1.1.6 in /opt/conda/lib/python3.7/site-packages (from neptune-client) (7.2.0)\r\n",
      "Requirement already satisfied: PyJWT in /opt/conda/lib/python3.7/site-packages (from neptune-client) (2.0.1)\r\n",
      "Requirement already satisfied: requests>=2.20.0 in /opt/conda/lib/python3.7/site-packages (from neptune-client) (2.25.1)\r\n",
      "Requirement already satisfied: requests-oauthlib>=1.0.0 in /opt/conda/lib/python3.7/site-packages (from neptune-client) (1.3.0)\r\n",
      "Requirement already satisfied: six>=1.12.0 in /opt/conda/lib/python3.7/site-packages (from neptune-client) (1.15.0)\r\n",
      "Requirement already satisfied: websocket-client>=0.35.0 in /opt/conda/lib/python3.7/site-packages (from neptune-client) (0.57.0)\r\n",
      "Requirement already satisfied: GitPython>=2.0.8 in /opt/conda/lib/python3.7/site-packages (from neptune-client) (3.1.13)\r\n",
      "Requirement already satisfied: packaging in /opt/conda/lib/python3.7/site-packages (from neptune-client) (20.9)\r\n",
      "Requirement already satisfied: urllib3 in /opt/conda/lib/python3.7/site-packages (from neptune-client) (1.26.3)\r\n",
      "Requirement already satisfied: gitdb<5,>=4.0.1 in /opt/conda/lib/python3.7/site-packages (from GitPython>=2.0.8->neptune-client) (4.0.5)\r\n",
      "Requirement already satisfied: smmap<4,>=3.0.1 in /opt/conda/lib/python3.7/site-packages (from gitdb<5,>=4.0.1->GitPython>=2.0.8->neptune-client) (3.0.5)\r\n",
      "Requirement already satisfied: idna<3,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests>=2.20.0->neptune-client) (2.10)\r\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests>=2.20.0->neptune-client) (2020.12.5)\r\n",
      "Requirement already satisfied: chardet<5,>=3.0.2 in /opt/conda/lib/python3.7/site-packages (from requests>=2.20.0->neptune-client) (3.0.4)\r\n",
      "Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.7/site-packages (from bravado->neptune-client) (3.7.4.3)\r\n",
      "Collecting bravado-core>=5.16.1\r\n",
      "  Downloading bravado_core-5.17.0-py2.py3-none-any.whl (67 kB)\r\n",
      "\u001b[K     |████████████████████████████████| 67 kB 1.0 MB/s \r\n",
      "\u001b[?25hRequirement already satisfied: pyyaml in /opt/conda/lib/python3.7/site-packages (from bravado->neptune-client) (5.3.1)\r\n",
      "Requirement already satisfied: msgpack in /opt/conda/lib/python3.7/site-packages (from bravado->neptune-client) (1.0.2)\r\n",
      "Collecting monotonic\r\n",
      "  Downloading monotonic-1.6-py2.py3-none-any.whl (8.2 kB)\r\n",
      "Requirement already satisfied: python-dateutil in /opt/conda/lib/python3.7/site-packages (from bravado->neptune-client) (2.8.1)\r\n",
      "Requirement already satisfied: simplejson in /opt/conda/lib/python3.7/site-packages (from bravado->neptune-client) (3.17.2)\r\n",
      "Collecting jsonref\r\n",
      "  Downloading jsonref-0.2-py3-none-any.whl (9.3 kB)\r\n",
      "Collecting swagger-spec-validator>=2.0.1\r\n",
      "  Downloading swagger_spec_validator-2.7.3-py2.py3-none-any.whl (27 kB)\r\n",
      "Requirement already satisfied: pytz in /opt/conda/lib/python3.7/site-packages (from bravado-core>=5.16.1->bravado->neptune-client) (2021.1)\r\n",
      "Requirement already satisfied: jsonschema[format]>=2.5.1 in /opt/conda/lib/python3.7/site-packages (from bravado-core>=5.16.1->bravado->neptune-client) (3.2.0)\r\n",
      "Requirement already satisfied: pyrsistent>=0.14.0 in /opt/conda/lib/python3.7/site-packages (from jsonschema[format]>=2.5.1->bravado-core>=5.16.1->bravado->neptune-client) (0.17.3)\r\n",
      "Requirement already satisfied: importlib-metadata in /opt/conda/lib/python3.7/site-packages (from jsonschema[format]>=2.5.1->bravado-core>=5.16.1->bravado->neptune-client) (3.4.0)\r\n",
      "Requirement already satisfied: setuptools in /opt/conda/lib/python3.7/site-packages (from jsonschema[format]>=2.5.1->bravado-core>=5.16.1->bravado->neptune-client) (49.6.0.post20210108)\r\n",
      "Requirement already satisfied: attrs>=17.4.0 in /opt/conda/lib/python3.7/site-packages (from jsonschema[format]>=2.5.1->bravado-core>=5.16.1->bravado->neptune-client) (20.3.0)\r\n",
      "Collecting rfc3987\r\n",
      "  Downloading rfc3987-1.3.8-py2.py3-none-any.whl (13 kB)\r\n",
      "Collecting webcolors\r\n",
      "  Downloading webcolors-1.11.1-py3-none-any.whl (9.9 kB)\r\n",
      "Collecting strict-rfc3339\r\n",
      "  Downloading strict-rfc3339-0.7.tar.gz (17 kB)\r\n",
      "Collecting jsonpointer>1.13\r\n",
      "  Downloading jsonpointer-2.1-py2.py3-none-any.whl (7.4 kB)\r\n",
      "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata->jsonschema[format]>=2.5.1->bravado-core>=5.16.1->bravado->neptune-client) (3.4.0)\r\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /opt/conda/lib/python3.7/site-packages (from packaging->neptune-client) (2.4.7)\r\n",
      "Requirement already satisfied: numpy>=1.15.4 in /opt/conda/lib/python3.7/site-packages (from pandas->neptune-client) (1.19.5)\r\n",
      "Building wheels for collected packages: neptune-client, strict-rfc3339\r\n",
      "  Building wheel for neptune-client (setup.py) ... \u001b[?25l-\b \b\\\b \b|\b \b/\b \bdone\r\n",
      "\u001b[?25h  Created wheel for neptune-client: filename=neptune_client-0.9.5-py2.py3-none-any.whl size=372957 sha256=d2e3d3723f3885e4886d1dee21cba7941b46d04418c5f63c01f22568496bcd6b\r\n",
      "  Stored in directory: /root/.cache/pip/wheels/46/7c/5e/365c23f03afdcf7d69e2c8b516f52c832f8d21a9004117c837\r\n",
      "  Building wheel for strict-rfc3339 (setup.py) ... \u001b[?25l-\b \b\\\b \bdone\r\n",
      "\u001b[?25h  Created wheel for strict-rfc3339: filename=strict_rfc3339-0.7-py3-none-any.whl size=18119 sha256=f3fedfa7456c5dc99f6cbce2d97ce76d80dc5bfe5d4af6242495f42ec8aea4d3\r\n",
      "  Stored in directory: /root/.cache/pip/wheels/f3/1d/9f/2a74caecb81b8beb9a4fbe1754203d4b7cf42ef5d39e0d2311\r\n",
      "Successfully built neptune-client strict-rfc3339\r\n",
      "Installing collected packages: webcolors, strict-rfc3339, rfc3987, jsonpointer, swagger-spec-validator, jsonref, monotonic, bravado-core, bravado, neptune-client\r\n",
      "Successfully installed bravado-11.0.3 bravado-core-5.17.0 jsonpointer-2.1 jsonref-0.2 monotonic-1.6 neptune-client-0.9.5 rfc3987-1.3.8 strict-rfc3339-0.7 swagger-spec-validator-2.7.3 webcolors-1.11.1\r\n"
     ]
    }
   ],
   "source": [
    "!pip install neptune-client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "social-coupon",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:28.227866Z",
     "iopub.status.busy": "2021-04-25T10:51:28.226623Z",
     "iopub.status.idle": "2021-04-25T10:51:34.621408Z",
     "shell.execute_reply": "2021-04-25T10:51:34.620372Z"
    },
    "id": "lYBoXTJxj56d",
    "outputId": "1946ec14-3d81-402b-b068-45a01c44e368",
    "papermill": {
     "duration": 6.478111,
     "end_time": "2021-04-25T10:51:34.621574",
     "exception": false,
     "start_time": "2021-04-25T10:51:28.143463",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# загружаем необходимые библиотеки\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.model_selection import GridSearchCV,RandomizedSearchCV\n",
    "from sklearn.preprocessing import OrdinalEncoder\n",
    "# импортируем функцию roc_auc_score() для вычисления AUC-ROC\n",
    "import optuna\n",
    "from optuna.samplers import RandomSampler\n",
    "from optuna.integration import OptunaSearchCV\n",
    "from optuna.distributions import IntUniformDistribution, UniformDistribution\n",
    "# импортируем класс StandardScaler, позволяющий\n",
    "# выполнить стандартизацию\n",
    "from sklearn.preprocessing import StandardScaler, LabelEncoder\n",
    "from category_encoders import TargetEncoder, OrdinalEncoder, LeaveOneOutEncoder, CatBoostEncoder\n",
    "import xgboost as xgb\n",
    "\n",
    "from sklearn.metrics import roc_auc_score, roc_curve \n",
    "from sklearn.model_selection import cross_val_predict\n",
    "import scikitplot as skplt\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "million-riding",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:34.768915Z",
     "iopub.status.busy": "2021-04-25T10:51:34.768275Z",
     "iopub.status.idle": "2021-04-25T10:51:35.141361Z",
     "shell.execute_reply": "2021-04-25T10:51:35.142208Z"
    },
    "id": "_CJf3w1Rj56e",
    "outputId": "c33766b2-a179-4a95-ea16-da37f6ef1116",
    "papermill": {
     "duration": 0.451849,
     "end_time": "2021-04-25T10:51:35.142399",
     "exception": false,
     "start_time": "2021-04-25T10:51:34.690550",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PassengerId</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Oconnor, Frankie</td>\n",
       "      <td>male</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>209245</td>\n",
       "      <td>27.14</td>\n",
       "      <td>C12239</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Bryan, Drew</td>\n",
       "      <td>male</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>27323</td>\n",
       "      <td>13.35</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Owens, Kenneth</td>\n",
       "      <td>male</td>\n",
       "      <td>0.33</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>CA 457703</td>\n",
       "      <td>71.29</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Kramer, James</td>\n",
       "      <td>male</td>\n",
       "      <td>19.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>A. 10866</td>\n",
       "      <td>13.04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Bond, Michael</td>\n",
       "      <td>male</td>\n",
       "      <td>25.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>427635</td>\n",
       "      <td>7.76</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99995</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>Bell, Adele</td>\n",
       "      <td>female</td>\n",
       "      <td>62.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 15008</td>\n",
       "      <td>14.86</td>\n",
       "      <td>D17243</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99996</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>Brown, Herman</td>\n",
       "      <td>male</td>\n",
       "      <td>66.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13273</td>\n",
       "      <td>11.15</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99997</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Childress, Charles</td>\n",
       "      <td>male</td>\n",
       "      <td>37.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.95</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99998</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Caughlin, Thomas</td>\n",
       "      <td>male</td>\n",
       "      <td>51.00</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>458654</td>\n",
       "      <td>30.92</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99999</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Enciso, Tyler</td>\n",
       "      <td>male</td>\n",
       "      <td>55.00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>458074</td>\n",
       "      <td>13.96</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100000 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             Survived  Pclass                Name     Sex    Age  SibSp  \\\n",
       "PassengerId                                                               \n",
       "0                   1       1    Oconnor, Frankie    male    NaN      2   \n",
       "1                   0       3         Bryan, Drew    male    NaN      0   \n",
       "2                   0       3      Owens, Kenneth    male   0.33      1   \n",
       "3                   0       3       Kramer, James    male  19.00      0   \n",
       "4                   1       3       Bond, Michael    male  25.00      0   \n",
       "...               ...     ...                 ...     ...    ...    ...   \n",
       "99995               1       2         Bell, Adele  female  62.00      0   \n",
       "99996               0       2       Brown, Herman    male  66.00      0   \n",
       "99997               0       3  Childress, Charles    male  37.00      0   \n",
       "99998               0       3    Caughlin, Thomas    male  51.00      0   \n",
       "99999               0       3       Enciso, Tyler    male  55.00      0   \n",
       "\n",
       "             Parch     Ticket   Fare   Cabin Embarked  \n",
       "PassengerId                                            \n",
       "0                0     209245  27.14  C12239        S  \n",
       "1                0      27323  13.35     NaN        S  \n",
       "2                2  CA 457703  71.29     NaN        S  \n",
       "3                0   A. 10866  13.04     NaN        S  \n",
       "4                0     427635   7.76     NaN        S  \n",
       "...            ...        ...    ...     ...      ...  \n",
       "99995            0   PC 15008  14.86  D17243        C  \n",
       "99996            0      13273  11.15     NaN        S  \n",
       "99997            0        NaN   9.95     NaN        S  \n",
       "99998            1     458654  30.92     NaN        S  \n",
       "99999            0     458074  13.96     NaN        S  \n",
       "\n",
       "[100000 rows x 11 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# увеличиваем количество выводимых столбцов\n",
    "pd.set_option('display.max_columns', 60)\n",
    "# загружаем набор\n",
    "data =  pd.read_csv(input_path / ('train.' + d_ext), index_col=INDEX_COL)\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "annual-exercise",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:35.289513Z",
     "iopub.status.busy": "2021-04-25T10:51:35.288360Z",
     "iopub.status.idle": "2021-04-25T10:51:35.297436Z",
     "shell.execute_reply": "2021-04-25T10:51:35.297958Z"
    },
    "papermill": {
     "duration": 0.085912,
     "end_time": "2021-04-25T10:51:35.298141",
     "exception": false,
     "start_time": "2021-04-25T10:51:35.212229",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    57226\n",
       "1    42774\n",
       "Name: Survived, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[TARGET].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "intermediate-proposition",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:35.448466Z",
     "iopub.status.busy": "2021-04-25T10:51:35.447594Z",
     "iopub.status.idle": "2021-04-25T10:51:35.451342Z",
     "shell.execute_reply": "2021-04-25T10:51:35.451842Z"
    },
    "id": "AaNAOFfSj56e",
    "papermill": {
     "duration": 0.082432,
     "end_time": "2021-04-25T10:51:35.451996",
     "exception": false,
     "start_time": "2021-04-25T10:51:35.369564",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if TEST:\n",
    "    data = data[:1000]\n",
    "    print(\"TEST MODE, Data shape = \",data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "religious-lending",
   "metadata": {
    "papermill": {
     "duration": 0.069714,
     "end_time": "2021-04-25T10:51:35.592156",
     "exception": false,
     "start_time": "2021-04-25T10:51:35.522442",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "monthly-imperial",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:35.737503Z",
     "iopub.status.busy": "2021-04-25T10:51:35.736617Z",
     "iopub.status.idle": "2021-04-25T10:51:36.047637Z",
     "shell.execute_reply": "2021-04-25T10:51:36.046365Z"
    },
    "papermill": {
     "duration": 0.3875,
     "end_time": "2021-04-25T10:51:36.047809",
     "exception": false,
     "start_time": "2021-04-25T10:51:35.660309",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# загружаем тестовый  набор\n",
    "test_o =  pd.read_csv(input_path / ('test.' + d_ext), index_col=INDEX_COL)\n",
    "# If selection of predictors\n",
    "if SELECT_PRED or SELECT_PRED2:\n",
    "    test_o = test[SELECTED_FEATURES]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "unusual-factory",
   "metadata": {
    "papermill": {
     "duration": 0.071244,
     "end_time": "2021-04-25T10:51:36.190650",
     "exception": false,
     "start_time": "2021-04-25T10:51:36.119406",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "portable-president",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:36.360543Z",
     "iopub.status.busy": "2021-04-25T10:51:36.347721Z",
     "iopub.status.idle": "2021-04-25T10:51:36.364677Z",
     "shell.execute_reply": "2021-04-25T10:51:36.365321Z"
    },
    "papermill": {
     "duration": 0.103167,
     "end_time": "2021-04-25T10:51:36.365548",
     "exception": false,
     "start_time": "2021-04-25T10:51:36.262381",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Index(['Survived', 'Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Ticket', 'Fare',\n",
       "        'Cabin', 'Embarked'],\n",
       "       dtype='object'),\n",
       " Index(['Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Ticket', 'Fare', 'Cabin',\n",
       "        'Embarked'],\n",
       "       dtype='object'))"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Удаление бесполезных переменных\n",
    "#data = data.drop(['Name','Ticket'],axis=1) #, inplace=True)\n",
    "#test_o = test_o.drop(['Name','Ticket'],axis=1)\n",
    "\n",
    "data = data.drop('Name',axis=1) #, inplace=True)\n",
    "test_o = test_o.drop('Name',axis=1)\n",
    "\n",
    "data.columns, test_o.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "imposed-black",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:36.516912Z",
     "iopub.status.busy": "2021-04-25T10:51:36.515697Z",
     "iopub.status.idle": "2021-04-25T10:51:36.897339Z",
     "shell.execute_reply": "2021-04-25T10:51:36.898681Z"
    },
    "papermill": {
     "duration": 0.463953,
     "end_time": "2021-04-25T10:51:36.898924",
     "exception": false,
     "start_time": "2021-04-25T10:51:36.434971",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(100000, 10) (100000, 10)\n"
     ]
    }
   ],
   "source": [
    "############## ref.: https://www.kaggle.com/jmargni/tps-apr-2021-lightgbm-cv\n",
    "############## https://www.kaggle.com/alexryzhkov/n3-tps-april-21-lightautoml-starter?scriptVersionId=59046001\n",
    "\n",
    "def create_extra_features(data):\n",
    "    # Age fillna with mean age for each class\n",
    "    age_map = data[['Age', 'Pclass']].dropna().groupby('Pclass').mean().to_dict()\n",
    "    data.Age = data.Age.fillna(data.Pclass.map(age_map['Age']))\n",
    "\n",
    "    # Cabin, fillna with 'X' and take first letter\n",
    "    data.Cabin = data.Cabin.map(lambda x: str(x)[0].strip())\n",
    "\n",
    "    # Ticket, fillna with 'X', split string and take first split \n",
    "    data.Ticket = data.Ticket.map(lambda x:str(x).split()[0] if len(str(x).split()) > 1 else 'X')\n",
    "\n",
    "    # Fare, fillna with mean value\n",
    "    # (THE ONLY FILLNA LEFT BECAUSE HERE WE USE TEST DATASET - LightAutoML can't do it in real life because of strict distinction between train and test stages)\n",
    "    data.Fare = data.Fare.fillna(data.Fare.mean())\n",
    "\n",
    "    # Embarked, fillna with 'X' value\n",
    "    data.Embarked = data.Embarked.fillna('X')\n",
    "\n",
    "    # Name, take only surnames\n",
    "    #data.Name = data.Name.map(lambda x: str(x).split(',')[0])\n",
    "    return data\n",
    "\n",
    "\n",
    "all_df = pd.concat([data, test_o]).reset_index(drop = True)\n",
    "all_df = create_extra_features(all_df)\n",
    "data, test_o = all_df[:len(data)], all_df[len(data):]\n",
    "print(data.shape, test_o.shape)\n",
    "\n",
    "test_o = test_o.drop('Survived',axis=1) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "tight-forest",
   "metadata": {
    "papermill": {
     "duration": 0.071656,
     "end_time": "2021-04-25T10:51:37.042071",
     "exception": false,
     "start_time": "2021-04-25T10:51:36.970415",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "смотрим дублирующие наблюдения:\n",
    "display(data[data.duplicated(keep=False)])\n",
    "\n",
    "удаляем дубли на месте, оставляя первое встретившееся наблюдение:\n",
    "data = data.drop_duplicates(subset=None,keep='first')\n",
    "\n",
    "смотрим снова дублирующие наблюдения:\n",
    "data[data.duplicated(keep=False)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "deadly-jaguar",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:37.194351Z",
     "iopub.status.busy": "2021-04-25T10:51:37.193310Z",
     "iopub.status.idle": "2021-04-25T10:51:37.197479Z",
     "shell.execute_reply": "2021-04-25T10:51:37.196826Z"
    },
    "papermill": {
     "duration": 0.084402,
     "end_time": "2021-04-25T10:51:37.197640",
     "exception": false,
     "start_time": "2021-04-25T10:51:37.113238",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# selection of predictors\n",
    "if SELECT_PRED:\n",
    "    # Separately, we selected the significant priediktors\n",
    "    # https://www.kaggle.com/c/tabular-playground-series-mar-2021/discussion/224530\n",
    "    # Variant 1:\n",
    "    SELECTED_FEATURES = [ # Noisy Cat Variables - 'cat5', 'cat7', 'cat8', 'cat10'\n",
    "                        'cat0', 'cat1', 'cat2', 'cat3', 'cat4', \n",
    "                        #'cat5', \n",
    "                        'cat6', \n",
    "                        #'cat7', 'cat8', \n",
    "                        'cat9', \n",
    "                        #'cat10', \n",
    "                        'cat11','cat12','cat13','cat14','cat15','cat16','cat17','cat18','cont0','cont1','cont2','cont3', \n",
    "                        'cont5', 'cont6', 'cont7', 'cont8', 'cont9', 'cont10']\n",
    "\n",
    "# selection of predictors\n",
    "if SELECT_PRED2:\n",
    "    # Separately, we selected the significant priediktors by the BoostARoota method\n",
    "    # Here they are, Variant 2 :\n",
    "    SELECTED_FEATURES = ['cat0', 'cat1', 'cat2', 'cat3', 'cat4', 'cat5', 'cat6', 'cat7', 'cat8', 'cat9', 'cat10', 'cat11', 'cat12', 'cat13', 'cat14', 'cat15', \n",
    "                         'cat16', 'cat17', 'cat18', 'cont0', 'cont1', 'cont2', 'cont3', \n",
    "                        #'cont4',\n",
    "                         'cont5', 'cont6', 'cont7', 'cont8', 'cont9', 'cont10']\n",
    "    \n",
    "if SELECT_PRED or SELECT_PRED2:\n",
    "    selfeat = SELECTED_FEATURES.copy()\n",
    "    selfeat.append(TARGET)\n",
    "    data = data[selfeat]\n",
    "    print(data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "alone-certification",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:37.350645Z",
     "iopub.status.busy": "2021-04-25T10:51:37.349622Z",
     "iopub.status.idle": "2021-04-25T10:51:37.439935Z",
     "shell.execute_reply": "2021-04-25T10:51:37.439352Z"
    },
    "id": "NUTcvj39j56f",
    "papermill": {
     "duration": 0.169801,
     "end_time": "2021-04-25T10:51:37.440107",
     "exception": false,
     "start_time": "2021-04-25T10:51:37.270306",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# создаем обучающий массив признаков, обучающий массив меток,\n",
    "# тестовый массив признаков, тестовый массив меток\n",
    "train, test, y_train, y_test = train_test_split(data.drop(TARGET, axis=1), \n",
    "                                                data[TARGET], \n",
    "                                                test_size=TEST_SIZE, \n",
    "                                                stratify=data[TARGET], \n",
    "                                                random_state=SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "stunning-worthy",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:37.599541Z",
     "iopub.status.busy": "2021-04-25T10:51:37.598469Z",
     "iopub.status.idle": "2021-04-25T10:51:37.605466Z",
     "shell.execute_reply": "2021-04-25T10:51:37.604902Z"
    },
    "id": "5yQ9Qp9Vj56f",
    "outputId": "1488f05d-b4f5-404c-8e95-8056ed6d6c2e",
    "papermill": {
     "duration": 0.092475,
     "end_time": "2021-04-25T10:51:37.605629",
     "exception": false,
     "start_time": "2021-04-25T10:51:37.513154",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# создаем список количественных признаков\n",
    "num_columns = train.select_dtypes(exclude='object').columns.tolist()\n",
    "# создаем списки категориальных признаков\n",
    "cat_columns = data.select_dtypes(include='object').columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "urban-parker",
   "metadata": {
    "id": "ydsW51MDj56g",
    "papermill": {
     "duration": 0.070321,
     "end_time": "2021-04-25T10:51:37.747471",
     "exception": false,
     "start_time": "2021-04-25T10:51:37.677150",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "> # ********  Phase 1 *************"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "freelance-annex",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:37.897911Z",
     "iopub.status.busy": "2021-04-25T10:51:37.896585Z",
     "iopub.status.idle": "2021-04-25T10:51:37.900242Z",
     "shell.execute_reply": "2021-04-25T10:51:37.899681Z"
    },
    "id": "ZWqH1pSAj56g",
    "papermill": {
     "duration": 0.080284,
     "end_time": "2021-04-25T10:51:37.900455",
     "exception": false,
     "start_time": "2021-04-25T10:51:37.820171",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# создаем конвейеры\n",
    "cat_pipe = Pipeline([('enc', \n",
    "                      TargetEncoder(),\n",
    "                      #OneHotEncoder(sparse=False, handle_unknown='ignore') \n",
    "                     )])\n",
    "num_pipe = Pipeline([('scal', None)]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "civil-roots",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:38.050524Z",
     "iopub.status.busy": "2021-04-25T10:51:38.048547Z",
     "iopub.status.idle": "2021-04-25T10:51:38.051231Z",
     "shell.execute_reply": "2021-04-25T10:51:38.051765Z"
    },
    "id": "-MiFAGrdj56g",
    "papermill": {
     "duration": 0.08017,
     "end_time": "2021-04-25T10:51:38.051910",
     "exception": false,
     "start_time": "2021-04-25T10:51:37.971740",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# создаем список трехэлементных кортежей, в котором\n",
    "# первый элемент кортежа - название конвейера с\n",
    "# преобразованиями для определенного типа признаков\n",
    "transformers = [('num', num_pipe, num_columns),\n",
    "                ('cat', cat_pipe, cat_columns)]\n",
    "\n",
    "# передаем список трансформеров в ColumnTransformer\n",
    "transformer = ColumnTransformer(transformers=transformers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "swiss-adoption",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:38.205451Z",
     "iopub.status.busy": "2021-04-25T10:51:38.203525Z",
     "iopub.status.idle": "2021-04-25T10:51:38.206229Z",
     "shell.execute_reply": "2021-04-25T10:51:38.206883Z"
    },
    "id": "Z1V4HwqWj56h",
    "papermill": {
     "duration": 0.0832,
     "end_time": "2021-04-25T10:51:38.207045",
     "exception": false,
     "start_time": "2021-04-25T10:51:38.123845",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if Phase_1:\n",
    "    params2 = params   \n",
    "    # задаем итоговый конвейер\n",
    "    ml_pipe = Pipeline([('tr', transformer), \n",
    "                        ('ml', xgb.XGBClassifier(**params) )])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "responsible-canvas",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:38.360061Z",
     "iopub.status.busy": "2021-04-25T10:51:38.359232Z",
     "iopub.status.idle": "2021-04-25T10:51:38.362318Z",
     "shell.execute_reply": "2021-04-25T10:51:38.362833Z"
    },
    "id": "BNzybeXGj56h",
    "papermill": {
     "duration": 0.082989,
     "end_time": "2021-04-25T10:51:38.362986",
     "exception": false,
     "start_time": "2021-04-25T10:51:38.279997",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if Phase_1:\n",
    "    # задаем сетку гиперпараметров\n",
    "    param_grid = [{\n",
    "                'tr__cat__enc': [#LabelEncoder(), \n",
    "                                 #OrdinalEncoder(),\n",
    "                                 #OneHotEncoder(sparse=False, handle_unknown='ignore'),\n",
    "                                 TargetEncoder(),\n",
    "                                 #CatBoostEncoder(),\n",
    "                                 #LeaveOneOutEncoder()\n",
    "                                ],\n",
    "                'tr__num__scal': [# StandardScaler(),\n",
    "                                  None\n",
    "                                 ],\n",
    "                'ml__learning_rate': RANG_LEARNING_RATE,\n",
    "                'ml__n_estimators' : N_ESTIMATORS_RANG\n",
    "    }]\n",
    "    \n",
    "    if TEST:\n",
    "            param_grid = [{\n",
    "                    'tr__cat__enc': [#TargetEncoder(),\n",
    "                                     LabelEncoder(), \n",
    "                                     OrdinalEncoder(),\n",
    "                                     OneHotEncoder(sparse=False, handle_unknown='ignore')\n",
    "                                    ],\n",
    "                    'tr__num__scal': [None,  StandardScaler()],\n",
    "                    'ml__learning_rate': [0.03,0.02],\n",
    "                    'ml__n_estimators' : [20,30]\n",
    "    }]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "fixed-settle",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:38.517649Z",
     "iopub.status.busy": "2021-04-25T10:51:38.516591Z",
     "iopub.status.idle": "2021-04-25T10:51:38.522067Z",
     "shell.execute_reply": "2021-04-25T10:51:38.521513Z"
    },
    "id": "toiOlZODj56h",
    "outputId": "4b8b1cc2-3607-4a94-aa12-a9f1b228a10d",
    "papermill": {
     "duration": 0.085583,
     "end_time": "2021-04-25T10:51:38.522242",
     "exception": false,
     "start_time": "2021-04-25T10:51:38.436659",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3 µs, sys: 1e+03 ns, total: 4 µs\n",
      "Wall time: 7.63 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "if Phase_1:\n",
    "    # создаем экземпляр класса RandomizedSearchCV или GridSearchCV, передав конвейер,\n",
    "    # сетку гиперпараметров и указав количество\n",
    "    # блоков перекрестной проверки, отключив запись метрик \n",
    "    # для обучающих блоков перекрестной проверки в атрибут cv_results_\n",
    "    #gs = GridSearchCV(\n",
    "    gs = RandomizedSearchCV( \n",
    "                      ml_pipe, \n",
    "                      param_grid, \n",
    "                      cv=CV, \n",
    "                      scoring='roc_auc', \n",
    "                      n_jobs=-1, \n",
    "                      return_train_score=False,\n",
    "                      verbose=3,\n",
    "                      random_state=SEED,\n",
    "                    n_iter = 100\n",
    "                     )\n",
    "    # выполняем решетчатый поиск\n",
    "    gs.fit(train, y_train)\n",
    "    # записываем оптимальные значения гиперпараметров\n",
    "    OPT_PARAMS_PIPE = gs.best_params_ "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "surface-advance",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:38.674454Z",
     "iopub.status.busy": "2021-04-25T10:51:38.673577Z",
     "iopub.status.idle": "2021-04-25T10:51:38.677448Z",
     "shell.execute_reply": "2021-04-25T10:51:38.676932Z"
    },
    "id": "wPDat-nmj56i",
    "papermill": {
     "duration": 0.081992,
     "end_time": "2021-04-25T10:51:38.677592",
     "exception": false,
     "start_time": "2021-04-25T10:51:38.595600",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if Phase_1:\n",
    "    y_probas = gs.predict_proba(test)\n",
    "    # смотрим наилучшие значения гиперпараметров\n",
    "    print('Наилучшие значения гиперпараметров: {}'.format(OPT_PARAMS_PIPE))\n",
    "    # смотрим наилучшее значение AUC\n",
    "    print('Наилучшее значение AUC: {:.3f}'.format(gs.best_score_))\n",
    "    # смотрим значение AUC на тестовой выборке\n",
    "    print('Значение AUC на тестовой выборке: {:.3f}'.format(\n",
    "        roc_auc_score(y_test, y_probas[:, 1] )))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bacterial-gathering",
   "metadata": {
    "papermill": {
     "duration": 0.07373,
     "end_time": "2021-04-25T10:51:38.823779",
     "exception": false,
     "start_time": "2021-04-25T10:51:38.750049",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "все FE без Name\n",
    "\n",
    "Наилучшие значения гиперпараметров: {'tr__num__scal': None, 'tr__cat__enc': OneHotEncoder(handle_unknown='ignore', sparse=False), 'ml__n_estimators': 500, 'ml__learning_rate': 0.0095}\n",
    "Наилучшее значение AUC: 0.852\n",
    "Значение AUC на тестовой выборке: 0.855"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "superb-daily",
   "metadata": {
    "papermill": {
     "duration": 0.071725,
     "end_time": "2021-04-25T10:51:38.968595",
     "exception": false,
     "start_time": "2021-04-25T10:51:38.896870",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Наилучшие значения гиперпараметров: {'tr__num__scal': None, 'tr__cat__enc': TargetEncoder(), 'ml__n_estimators': 500, 'ml__learning_rate': 0.0095}\n",
    "Наилучшее значение AUC: 0.853\n",
    "Значение AUC на тестовой выборке: 0.855"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "handy-canadian",
   "metadata": {
    "papermill": {
     "duration": 0.071332,
     "end_time": "2021-04-25T10:51:39.112757",
     "exception": false,
     "start_time": "2021-04-25T10:51:39.041425",
     "status": "completed"
    },
    "tags": []
   },
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "rising-broadcasting",
   "metadata": {
    "papermill": {
     "duration": 0.071865,
     "end_time": "2021-04-25T10:51:39.257823",
     "exception": false,
     "start_time": "2021-04-25T10:51:39.185958",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Наилучшие значения гиперпараметров: {'tr__num__scal': None, 'tr__cat__enc': OrdinalEncoder(), 'ml__n_estimators': 500, 'ml__learning_rate': 0.005}\n",
    "Наилучшее значение AUC: 0.850\n",
    "Значение AUC на тестовой выборке: 0.853\n",
    "add Codeadd Markdown"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "shaped-miller",
   "metadata": {
    "papermill": {
     "duration": 0.073166,
     "end_time": "2021-04-25T10:51:39.404400",
     "exception": false,
     "start_time": "2021-04-25T10:51:39.331234",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Наилучшие значения гиперпараметров: {'tr__num__scal': None, 'tr__cat__enc': OrdinalEncoder(), 'ml__n_estimators': 500, 'ml__learning_rate': 0.005}\n",
    "Наилучшее значение AUC: 0.841\n",
    "Значение AUC на тестовой выборке: 0.843"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "developed-instruction",
   "metadata": {
    "papermill": {
     "duration": 0.081366,
     "end_time": "2021-04-25T10:51:39.558716",
     "exception": false,
     "start_time": "2021-04-25T10:51:39.477350",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Наилучшие значения гиперпараметров: {'tr__num__scal': None, 'tr__cat__enc': OneHotEncoder(handle_unknown='ignore', sparse=False), 'ml__n_estimators': 500, 'ml__learning_rate': 0.005}\n",
    "Наилучшее значение AUC: 0.841\n",
    "Значение AUC на тестовой выборке: 0.843"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "serial-socket",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:39.723337Z",
     "iopub.status.busy": "2021-04-25T10:51:39.722234Z",
     "iopub.status.idle": "2021-04-25T10:51:39.725639Z",
     "shell.execute_reply": "2021-04-25T10:51:39.726200Z"
    },
    "id": "zCumPqG-j56i",
    "papermill": {
     "duration": 0.089457,
     "end_time": "2021-04-25T10:51:39.726377",
     "exception": false,
     "start_time": "2021-04-25T10:51:39.636920",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if Phase_1:\n",
    "    # увеличиваем количество выводимых строк\n",
    "    pd.set_option('display.max_rows', 300)\n",
    "\n",
    "    # выводим результаты\n",
    "    cv_results = pd.DataFrame(gs.cv_results_)[['mean_test_score',\n",
    "                                               'param_ml__learning_rate',\n",
    "                                               'param_tr__cat__enc',\n",
    "                                               'param_tr__num__scal'\n",
    "                                               ]]\n",
    "    cv_results = cv_results.sort_values(by='mean_test_score', ascending=False).reset_index(drop=True)\n",
    "\n",
    "    for col in cv_results.columns[cv_results.columns.str.contains('enc')]:\n",
    "        if col in 'mean_test_score':\n",
    "            continue\n",
    "\n",
    "        cv_results[col] = cv_results[col].apply(lambda x: type(x).__name__)\n",
    "\n",
    "    display(cv_results)\n",
    "    \n",
    "    with open(NAME_RUN_LOG, 'at', encoding='utf-8') as logfile:\n",
    "        print('Временная метка окончания ФАЗЫ 1 : {}'.format(datetime.datetime.now().strftime(\"%d-%m-%Y %H:%M\")), \n",
    "          file=logfile)\n",
    "        print('Наилучшие значения гиперпараметров: {}'.format(gs.best_params_), file=logfile)\n",
    "        print('Наилучшее значение AUC: {:.3f}'.format(gs.best_score_), file=logfile)\n",
    "        print('Значение AUC на тестовой выборке: {:.3f}'.format(\n",
    "            roc_auc_score(y_test, gs.predict_proba(test)[:, 1])), file=logfile)\n",
    "        print('Таблица значений: ', file=logfile)\n",
    "        #print(display(cv_results), file=logfile)\n",
    "    with open(NAME_RUN_LOG, 'at', encoding='utf-8') as logfile:\n",
    "        cv_results.to_csv(NAME_RUN_LOG, index=False, mode='at')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "downtown-reminder",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:39.881137Z",
     "iopub.status.busy": "2021-04-25T10:51:39.880184Z",
     "iopub.status.idle": "2021-04-25T10:51:39.884202Z",
     "shell.execute_reply": "2021-04-25T10:51:39.883194Z"
    },
    "papermill": {
     "duration": 0.081485,
     "end_time": "2021-04-25T10:51:39.884362",
     "exception": false,
     "start_time": "2021-04-25T10:51:39.802877",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if Phase_1:\n",
    "    # рисуем ROC кривую \n",
    "    skplt.metrics.plot_roc(y_test, y_probas)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "confirmed-praise",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:40.040724Z",
     "iopub.status.busy": "2021-04-25T10:51:40.039562Z",
     "iopub.status.idle": "2021-04-25T10:51:40.044855Z",
     "shell.execute_reply": "2021-04-25T10:51:40.044320Z"
    },
    "papermill": {
     "duration": 0.087714,
     "end_time": "2021-04-25T10:51:40.044990",
     "exception": false,
     "start_time": "2021-04-25T10:51:39.957276",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.47231523177731743"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if Phase_1:\n",
    "    # и выбираем лучшую точку порога\n",
    "    def Find_Optimal_Cutoff(target, predicted):\n",
    "        fpr, tpr, threshold = roc_curve(target, predicted)\n",
    "        i = np.arange(len(tpr)) \n",
    "        roc = pd.DataFrame({'tf' : pd.Series(tpr-(1-fpr), index=i), 'threshold' : pd.Series(threshold, index=i)})\n",
    "        roc_t = roc.loc[(roc.tf-0).abs().argsort()[:1]]\n",
    "\n",
    "        return list(roc_t['threshold']) \n",
    "\n",
    "    threshold = Find_Optimal_Cutoff(y_test, y_probas[:, 1])\n",
    "\n",
    "threshold"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "metallic-fleet",
   "metadata": {
    "id": "1CNKuN-jj56i",
    "papermill": {
     "duration": 0.075618,
     "end_time": "2021-04-25T10:51:40.195204",
     "exception": false,
     "start_time": "2021-04-25T10:51:40.119586",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "> # ********  Phase 2 *************"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "unlikely-cancellation",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:40.353035Z",
     "iopub.status.busy": "2021-04-25T10:51:40.351809Z",
     "iopub.status.idle": "2021-04-25T10:51:40.355640Z",
     "shell.execute_reply": "2021-04-25T10:51:40.355028Z"
    },
    "id": "itTSx5MTj56j",
    "papermill": {
     "duration": 0.085734,
     "end_time": "2021-04-25T10:51:40.355769",
     "exception": false,
     "start_time": "2021-04-25T10:51:40.270035",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if Phase_2 or Phase_3 or Phase_4:\n",
    "    # трансформеры\n",
    "    upd_cat_pipe = Pipeline([('enc',TargetEncoder() )])  #OneHotEncoder(handle_unknown='ignore', sparse=False))])\n",
    "    upd_num_pipe = Pipeline([('scal', None)]) # \n",
    "\n",
    "    upd_transformers = [('num', upd_num_pipe, num_columns),\n",
    "                        ('cat', upd_cat_pipe, cat_columns),]\n",
    "\n",
    "    # передаем список трансформеров в ColumnTransformer\n",
    "    upd_transformer = ColumnTransformer(transformers=upd_transformers)\n",
    "\n",
    "    # задаем итоговый конвейер ПОКА ЧТО С НАЧАЛЬНЫМИ НАСТРОЙКАМИ\n",
    "    ml_pipe2 = Pipeline([('tr', upd_transformer), \n",
    "                         ('ml', xgb.XGBClassifier(**params) )])\n",
    "\n",
    "    # УСТАНОВКА В НОВЫЙ PIPE НАИЛУЧШИХ ЗНАЧЕНИЙ НАЙДЕННЫХ В ПРЕДЫДУЩЕЙ ФАЗЕ:  \n",
    "    if Phase_1:  \n",
    "        # присваиваем итоговому конвейеру оптимальные значения гиперпараметров\n",
    "        print('Before modification ml__learning_rate ml_pipe2: ', ml_pipe2.get_params()['ml__learning_rate']) # 'n_estimators'\n",
    "        ml_pipe2.set_params(**OPT_PARAMS_PIPE)\n",
    "        print('After modification ml__learning_rate ml_pipe2: ', ml_pipe2.get_params()['ml__learning_rate'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "light-steering",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:40.511370Z",
     "iopub.status.busy": "2021-04-25T10:51:40.510416Z",
     "iopub.status.idle": "2021-04-25T10:51:40.514366Z",
     "shell.execute_reply": "2021-04-25T10:51:40.513745Z"
    },
    "papermill": {
     "duration": 0.083289,
     "end_time": "2021-04-25T10:51:40.514563",
     "exception": false,
     "start_time": "2021-04-25T10:51:40.431274",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "params2 = params3 = params4 = {}\n",
    "\n",
    "if Phase_2: \n",
    "    params2 = {\n",
    "        #'ml__use_label_encoder' : False ,\n",
    "        # (1) гиперпараметры, задающие структуру дерева:\n",
    "        #'tree_method' : 'gpu_hist' , \n",
    "        'ml__max_depth' : IntUniformDistribution(1 , 20),\n",
    "        'ml__min_child_weight' : IntUniformDistribution(1 , 300),\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "introductory-courtesy",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:40.672603Z",
     "iopub.status.busy": "2021-04-25T10:51:40.671664Z",
     "iopub.status.idle": "2021-04-25T10:51:40.675619Z",
     "shell.execute_reply": "2021-04-25T10:51:40.675078Z"
    },
    "papermill": {
     "duration": 0.085211,
     "end_time": "2021-04-25T10:51:40.675756",
     "exception": false,
     "start_time": "2021-04-25T10:51:40.590545",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    " # (2) гиперпараметры, привносящие случайность в ходе построения дерева:\n",
    "if Phase_3: \n",
    "    params3 = {\n",
    "              'ml__colsample_bytree' : UniformDistribution( 0 , 1.0),\n",
    "              'ml__subsample' : UniformDistribution( 0 , 1.0)\n",
    "             }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "respected-dance",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:40.835676Z",
     "iopub.status.busy": "2021-04-25T10:51:40.834622Z",
     "iopub.status.idle": "2021-04-25T10:51:40.838609Z",
     "shell.execute_reply": "2021-04-25T10:51:40.838049Z"
    },
    "papermill": {
     "duration": 0.087288,
     "end_time": "2021-04-25T10:51:40.838808",
     "exception": false,
     "start_time": "2021-04-25T10:51:40.751520",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# (3) гиперпараметры, задающие регуляризацию функции потерь:\n",
    "if Phase_4: \n",
    "    params4 = {\n",
    "            'ml__lambda' : UniformDistribution( 1e-5 , 1.0),\n",
    "            'ml__alpha' : UniformDistribution( 1e-5 , 1.0),\n",
    "            'ml__gamma' : UniformDistribution( 1e-5 , 1.0)\n",
    "             }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "sized-minnesota",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:40.995605Z",
     "iopub.status.busy": "2021-04-25T10:51:40.994588Z",
     "iopub.status.idle": "2021-04-25T10:51:40.998346Z",
     "shell.execute_reply": "2021-04-25T10:51:40.997785Z"
    },
    "papermill": {
     "duration": 0.08398,
     "end_time": "2021-04-25T10:51:40.998493",
     "exception": false,
     "start_time": "2021-04-25T10:51:40.914513",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if Phase_2 or Phase_3 or Phase_4:  \n",
    "    # задаем новое пространство поиска\n",
    "    param_distributions = {**params2,**params3,**params4}\n",
    "    print(param_distributions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "russian-matrix",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:41.158143Z",
     "iopub.status.busy": "2021-04-25T10:51:41.157019Z",
     "iopub.status.idle": "2021-04-25T10:51:41.160237Z",
     "shell.execute_reply": "2021-04-25T10:51:41.160785Z"
    },
    "id": "VZ8ujEQBj56j",
    "papermill": {
     "duration": 0.086815,
     "end_time": "2021-04-25T10:51:41.160994",
     "exception": false,
     "start_time": "2021-04-25T10:51:41.074179",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if Phase_2 or Phase_3 or Phase_4:  \n",
    "    # задаем новое пространство поиска\n",
    "    #param_distributions = {**params2,**params3,**params4}\n",
    " \n",
    "    if TEST:\n",
    "        param_distributions = {'ml__max_depth' : IntUniformDistribution(11, 13),\n",
    "                              'ml__min_child_weight' : IntUniformDistribution(23, 25),}  \n",
    "\n",
    "    # создаем экземпляр класса OptunaSearchCV\n",
    "    optuna_search = OptunaSearchCV(\n",
    "        ml_pipe2,\n",
    "        param_distributions,\n",
    "        scoring='roc_auc',\n",
    "        n_trials=N_TRIALS,\n",
    "        n_jobs=-1,\n",
    "        refit=True,\n",
    "        #verbose=1,\n",
    "        cv=CV_O,\n",
    "        random_state=SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "sticky-offer",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:41.320724Z",
     "iopub.status.busy": "2021-04-25T10:51:41.319614Z",
     "iopub.status.idle": "2021-04-25T10:51:41.325871Z",
     "shell.execute_reply": "2021-04-25T10:51:41.324961Z"
    },
    "id": "3Tbb9_ryj56k",
    "papermill": {
     "duration": 0.087109,
     "end_time": "2021-04-25T10:51:41.326081",
     "exception": false,
     "start_time": "2021-04-25T10:51:41.238972",
     "status": "completed"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4 µs, sys: 0 ns, total: 4 µs\n",
      "Wall time: 7.87 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "if Phase_2 or Phase_3 or Phase_4:  \n",
    "    # выполняем оптимизацию\n",
    "    optuna_search.fit(train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "suspected-investor",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:41.488722Z",
     "iopub.status.busy": "2021-04-25T10:51:41.487600Z",
     "iopub.status.idle": "2021-04-25T10:51:41.491463Z",
     "shell.execute_reply": "2021-04-25T10:51:41.490847Z"
    },
    "id": "SyreK__Qj56l",
    "papermill": {
     "duration": 0.088597,
     "end_time": "2021-04-25T10:51:41.491593",
     "exception": false,
     "start_time": "2021-04-25T10:51:41.402996",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if Phase_2 or Phase_3 or Phase_4:  \n",
    "    # записываем оптимальные значения гиперпараметров optuna\n",
    "    opt_optuna_params_pipe = optuna_search.best_params_\n",
    "    if Phase_1:\n",
    "        opt_optuna_params_pipe['ml__learning_rate'] = OPT_PARAMS_PIPE['ml__learning_rate']\n",
    "        opt_optuna_params_pipe['ml__n_estimators'] = OPT_PARAMS_PIPE['ml__n_estimators']\n",
    "\n",
    "    # печатаем наилучшие значения гиперпараметров\n",
    "    print('Наилучшие значения гиперпараметров: {}'.format(opt_optuna_params_pipe))\n",
    "    # печатаем наилучшее значение AUC\n",
    "    print('Наилучшее значение AUC: {:.3f}'.format(optuna_search.best_score_))\n",
    "    # смотрим значение AUC на тестовой выборке\n",
    "    print('AUC на тестовой выборке: {:.3f}'.format(\n",
    "        roc_auc_score(y_test, optuna_search.predict_proba(test)[:, 1])))\n",
    "    print('')\n",
    "    with open(NAME_RUN_LOG, 'at', encoding='utf-8') as logfile:\n",
    "        print('/nВременная метка окончания ФАЗЫ 2 : {}'.format(datetime.datetime.now().strftime(\"%d-%m-%Y %H:%M\")), \n",
    "          file=logfile)\n",
    "        print('Наилучшие значения гиперпараметров найденых optuna: \\n{}'.format(opt_optuna_params_pipe), file=logfile)\n",
    "        print('Наилучшее значение AUC: {:.3f}'.format(optuna_search.best_score_), file=logfile)\n",
    "        print('AUC на тестовой выборке: {:.3f}\\n'.format(\n",
    "            roc_auc_score(y_test, optuna_search.predict_proba(test)[:, 1])), file=logfile)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cardiovascular-conservative",
   "metadata": {
    "id": "XEUoguQDj56m",
    "papermill": {
     "duration": 0.074747,
     "end_time": "2021-04-25T10:51:41.642008",
     "exception": false,
     "start_time": "2021-04-25T10:51:41.567261",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "> # ********  Final phase *************"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "outdoor-feeding",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:41.797081Z",
     "iopub.status.busy": "2021-04-25T10:51:41.796115Z",
     "iopub.status.idle": "2021-04-25T10:51:41.799982Z",
     "shell.execute_reply": "2021-04-25T10:51:41.799434Z"
    },
    "papermill": {
     "duration": 0.084102,
     "end_time": "2021-04-25T10:51:41.800134",
     "exception": false,
     "start_time": "2021-04-25T10:51:41.716032",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# high parameters at the end\n",
    "if HIGH_PARAM_END:\n",
    "        opt_optuna_params_pipe['ml__learning_rate'] = 20000\n",
    "        opt_optuna_params_pipe['ml__n_estimators'] = 0.0021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "mobile-entity",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:41.958120Z",
     "iopub.status.busy": "2021-04-25T10:51:41.956950Z",
     "iopub.status.idle": "2021-04-25T10:51:41.964255Z",
     "shell.execute_reply": "2021-04-25T10:51:41.963410Z"
    },
    "id": "iexbKXzIj56m",
    "papermill": {
     "duration": 0.088831,
     "end_time": "2021-04-25T10:51:41.964409",
     "exception": false,
     "start_time": "2021-04-25T10:51:41.875578",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# формируем полный массив меток и массив признаков\n",
    "# формируем полный массив тренировочных данных\n",
    "fulldata = data.drop(TARGET, axis=1) \n",
    "# формируем полный массив меток и массив признаков  \n",
    "y_fulldata = data[TARGET] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "imported-million",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:42.124859Z",
     "iopub.status.busy": "2021-04-25T10:51:42.123813Z",
     "iopub.status.idle": "2021-04-25T10:51:42.127414Z",
     "shell.execute_reply": "2021-04-25T10:51:42.126872Z"
    },
    "id": "JEGiSIpDj56m",
    "papermill": {
     "duration": 0.086963,
     "end_time": "2021-04-25T10:51:42.127554",
     "exception": false,
     "start_time": "2021-04-25T10:51:42.040591",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#  трансформеры\n",
    "#upd_cat_pipe = Pipeline([('enc', OneHotEncoder(handle_unknown='ignore', sparse=False))])\n",
    "#upd_num_pipe = Pipeline([('scal', None)]) \n",
    "# список трансформеров\n",
    "#upd_transformers = [('num', upd_num_pipe, num_columns),('cat', upd_cat_pipe, cat_columns),]\n",
    "\n",
    "# передаем список трансформеров в ColumnTransformer\n",
    "#upd_transformer = ColumnTransformer(transformers=upd_transformers)\n",
    "    \n",
    "# задаем итоговый конвейер ПОКА ЧТО С НАЧАЛЬНЫМИ НАСТРОЙКАМИ\n",
    "ml_pipe3 = Pipeline([('tr', transformer), \n",
    "                     ('ml', xgb.XGBClassifier(**params) )])\n",
    "\n",
    "if Phase_1:\n",
    "    ml_pipe3.set_params(**OPT_PARAMS_PIPE)\n",
    "\n",
    "\n",
    "if Phase_2 or Phase_3 or Phase_4:\n",
    "    # Для контроля изменений смотрим выбранные параметры модели \n",
    "    opt_par = ['ml__n_estimators','ml__learning_rate',\n",
    "               'ml__max_depth','ml__min_child_weight','ml__subsample']\n",
    "    print(opt_par)\n",
    "    print( 'Before modification ml_pipe3: ' ,[ml_pipe3.get_params()[x] for x in opt_par])\n",
    "    \n",
    "    # УСТАНОВКА В НОВЫЙ PIPE НАИЛУЧШИХ ЗНАЧЕНИЙ НАЙДЕННЫХ В ПРЕДЫДУЩИХ ФАЗАХ:  \n",
    "    # присваиваем итоговому конвейеру оптимальные значения гиперпараметров найденные optuna\n",
    "    ml_pipe3.set_params(**opt_optuna_params_pipe)\n",
    "    \n",
    "    # Для контроля изменений\n",
    "    print( 'After modification ml_pipe3: ' ,[ml_pipe3.get_params()[x] for x in opt_par])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "square-entity",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:42.286375Z",
     "iopub.status.busy": "2021-04-25T10:51:42.285294Z",
     "iopub.status.idle": "2021-04-25T10:51:42.290200Z",
     "shell.execute_reply": "2021-04-25T10:51:42.289660Z"
    },
    "papermill": {
     "duration": 0.087365,
     "end_time": "2021-04-25T10:51:42.290336",
     "exception": false,
     "start_time": "2021-04-25T10:51:42.202971",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20000"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ml_pipe3.get_params()['ml__n_estimators']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "numerical-biology",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:42.451499Z",
     "iopub.status.busy": "2021-04-25T10:51:42.450318Z",
     "iopub.status.idle": "2021-04-25T10:51:42.455519Z",
     "shell.execute_reply": "2021-04-25T10:51:42.454949Z"
    },
    "papermill": {
     "duration": 0.087622,
     "end_time": "2021-04-25T10:51:42.455659",
     "exception": false,
     "start_time": "2021-04-25T10:51:42.368037",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ml__n_estimators': 5000}"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "par = {'ml__n_estimators': 5000}\n",
    "par"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "deadly-queue",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:42.617688Z",
     "iopub.status.busy": "2021-04-25T10:51:42.616671Z",
     "iopub.status.idle": "2021-04-25T10:51:42.620462Z",
     "shell.execute_reply": "2021-04-25T10:51:42.619910Z"
    },
    "papermill": {
     "duration": 0.086094,
     "end_time": "2021-04-25T10:51:42.620632",
     "exception": false,
     "start_time": "2021-04-25T10:51:42.534538",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "par['ml__n_estimators'] = int(ml_pipe3.get_params()['ml__n_estimators'] * (1 + 1/CV))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "national-intermediate",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:42.781604Z",
     "iopub.status.busy": "2021-04-25T10:51:42.780436Z",
     "iopub.status.idle": "2021-04-25T10:51:42.785319Z",
     "shell.execute_reply": "2021-04-25T10:51:42.785946Z"
    },
    "papermill": {
     "duration": 0.087861,
     "end_time": "2021-04-25T10:51:42.786138",
     "exception": false,
     "start_time": "2021-04-25T10:51:42.698277",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ml__n_estimators': 24000}"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "par"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "adequate-dialogue",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:42.950132Z",
     "iopub.status.busy": "2021-04-25T10:51:42.949253Z",
     "iopub.status.idle": "2021-04-25T10:51:42.972202Z",
     "shell.execute_reply": "2021-04-25T10:51:42.971595Z"
    },
    "papermill": {
     "duration": 0.106885,
     "end_time": "2021-04-25T10:51:42.972334",
     "exception": false,
     "start_time": "2021-04-25T10:51:42.865449",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'memory': None,\n",
       " 'steps': [('tr',\n",
       "   ColumnTransformer(transformers=[('num', Pipeline(steps=[('scal', None)]),\n",
       "                                    ['Pclass', 'Age', 'SibSp', 'Parch', 'Fare']),\n",
       "                                   ('cat',\n",
       "                                    Pipeline(steps=[('enc', TargetEncoder())]),\n",
       "                                    Index(['Sex', 'Ticket', 'Cabin', 'Embarked'], dtype='object'))])),\n",
       "  ('ml',\n",
       "   XGBClassifier(alpha=0.32470257316374473, base_score=None, booster=None,\n",
       "                 colsample_bylevel=None, colsample_bynode=None,\n",
       "                 colsample_bytree=0.6248628797707747, device='gpu',\n",
       "                 gamma=0.9123913042030198, gpu_device_id=0, gpu_id=None,\n",
       "                 gpu_platform_id=0, importance_type='gain',\n",
       "                 interaction_constraints=None, lambda=0.6290623246094361,\n",
       "                 learning_rate=0.0005, max_delta_step=None, max_depth=6,\n",
       "                 min_child_weight=24, missing=nan, monotone_constraints=None,\n",
       "                 n_estimators=20000, n_jobs=None, num_parallel_tree=None,\n",
       "                 predictor='gpu_predictor', random_state=2599520, reg_alpha=None,\n",
       "                 reg_lambda=None, scale_pos_weight=None,\n",
       "                 subsample=0.7109465438773837, tree_method='gpu_hist', ...))],\n",
       " 'verbose': False,\n",
       " 'tr': ColumnTransformer(transformers=[('num', Pipeline(steps=[('scal', None)]),\n",
       "                                  ['Pclass', 'Age', 'SibSp', 'Parch', 'Fare']),\n",
       "                                 ('cat',\n",
       "                                  Pipeline(steps=[('enc', TargetEncoder())]),\n",
       "                                  Index(['Sex', 'Ticket', 'Cabin', 'Embarked'], dtype='object'))]),\n",
       " 'ml': XGBClassifier(alpha=0.32470257316374473, base_score=None, booster=None,\n",
       "               colsample_bylevel=None, colsample_bynode=None,\n",
       "               colsample_bytree=0.6248628797707747, device='gpu',\n",
       "               gamma=0.9123913042030198, gpu_device_id=0, gpu_id=None,\n",
       "               gpu_platform_id=0, importance_type='gain',\n",
       "               interaction_constraints=None, lambda=0.6290623246094361,\n",
       "               learning_rate=0.0005, max_delta_step=None, max_depth=6,\n",
       "               min_child_weight=24, missing=nan, monotone_constraints=None,\n",
       "               n_estimators=20000, n_jobs=None, num_parallel_tree=None,\n",
       "               predictor='gpu_predictor', random_state=2599520, reg_alpha=None,\n",
       "               reg_lambda=None, scale_pos_weight=None,\n",
       "               subsample=0.7109465438773837, tree_method='gpu_hist', ...),\n",
       " 'tr__n_jobs': None,\n",
       " 'tr__remainder': 'drop',\n",
       " 'tr__sparse_threshold': 0.3,\n",
       " 'tr__transformer_weights': None,\n",
       " 'tr__transformers': [('num',\n",
       "   Pipeline(steps=[('scal', None)]),\n",
       "   ['Pclass', 'Age', 'SibSp', 'Parch', 'Fare']),\n",
       "  ('cat',\n",
       "   Pipeline(steps=[('enc', TargetEncoder())]),\n",
       "   Index(['Sex', 'Ticket', 'Cabin', 'Embarked'], dtype='object'))],\n",
       " 'tr__verbose': False,\n",
       " 'tr__num': Pipeline(steps=[('scal', None)]),\n",
       " 'tr__cat': Pipeline(steps=[('enc', TargetEncoder())]),\n",
       " 'tr__num__memory': None,\n",
       " 'tr__num__steps': [('scal', None)],\n",
       " 'tr__num__verbose': False,\n",
       " 'tr__num__scal': None,\n",
       " 'tr__cat__memory': None,\n",
       " 'tr__cat__steps': [('enc', TargetEncoder())],\n",
       " 'tr__cat__verbose': False,\n",
       " 'tr__cat__enc': TargetEncoder(),\n",
       " 'tr__cat__enc__cols': None,\n",
       " 'tr__cat__enc__drop_invariant': False,\n",
       " 'tr__cat__enc__handle_missing': 'value',\n",
       " 'tr__cat__enc__handle_unknown': 'value',\n",
       " 'tr__cat__enc__min_samples_leaf': 1,\n",
       " 'tr__cat__enc__return_df': True,\n",
       " 'tr__cat__enc__smoothing': 1.0,\n",
       " 'tr__cat__enc__verbose': 0,\n",
       " 'ml__objective': 'binary:logistic',\n",
       " 'ml__use_label_encoder': True,\n",
       " 'ml__base_score': None,\n",
       " 'ml__booster': None,\n",
       " 'ml__colsample_bylevel': None,\n",
       " 'ml__colsample_bynode': None,\n",
       " 'ml__colsample_bytree': 0.6248628797707747,\n",
       " 'ml__gamma': 0.9123913042030198,\n",
       " 'ml__gpu_id': None,\n",
       " 'ml__importance_type': 'gain',\n",
       " 'ml__interaction_constraints': None,\n",
       " 'ml__learning_rate': 0.0005,\n",
       " 'ml__max_delta_step': None,\n",
       " 'ml__max_depth': 6,\n",
       " 'ml__min_child_weight': 24,\n",
       " 'ml__missing': nan,\n",
       " 'ml__monotone_constraints': None,\n",
       " 'ml__n_estimators': 20000,\n",
       " 'ml__n_jobs': None,\n",
       " 'ml__num_parallel_tree': None,\n",
       " 'ml__random_state': 2599520,\n",
       " 'ml__reg_alpha': None,\n",
       " 'ml__reg_lambda': None,\n",
       " 'ml__scale_pos_weight': None,\n",
       " 'ml__subsample': 0.7109465438773837,\n",
       " 'ml__tree_method': 'gpu_hist',\n",
       " 'ml__validate_parameters': None,\n",
       " 'ml__verbosity': None,\n",
       " 'ml__lambda': 0.6290623246094361,\n",
       " 'ml__alpha': 0.32470257316374473,\n",
       " 'ml__device': 'gpu',\n",
       " 'ml__gpu_platform_id': 0,\n",
       " 'ml__gpu_device_id': 0,\n",
       " 'ml__predictor': 'gpu_predictor'}"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ml_pipe3.get_params()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "nasty-humor",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:43.155999Z",
     "iopub.status.busy": "2021-04-25T10:51:43.142543Z",
     "iopub.status.idle": "2021-04-25T10:51:43.161301Z",
     "shell.execute_reply": "2021-04-25T10:51:43.160682Z"
    },
    "papermill": {
     "duration": 0.111017,
     "end_time": "2021-04-25T10:51:43.161483",
     "exception": false,
     "start_time": "2021-04-25T10:51:43.050466",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('tr',\n",
       "                 ColumnTransformer(transformers=[('num',\n",
       "                                                  Pipeline(steps=[('scal',\n",
       "                                                                   None)]),\n",
       "                                                  ['Pclass', 'Age', 'SibSp',\n",
       "                                                   'Parch', 'Fare']),\n",
       "                                                 ('cat',\n",
       "                                                  Pipeline(steps=[('enc',\n",
       "                                                                   TargetEncoder())]),\n",
       "                                                  Index(['Sex', 'Ticket', 'Cabin', 'Embarked'], dtype='object'))])),\n",
       "                ('ml',\n",
       "                 XGBClassifier(alpha=0.32470257316374473, base_score=None,\n",
       "                               booster=None, colsample_bylevel=None,\n",
       "                               colsamp...\n",
       "                               lambda=0.6290623246094361, learning_rate=0.0005,\n",
       "                               max_delta_step=None, max_depth=6,\n",
       "                               min_child_weight=24, missing=nan,\n",
       "                               monotone_constraints=None, n_estimators=24000,\n",
       "                               n_jobs=None, num_parallel_tree=None,\n",
       "                               predictor='gpu_predictor', random_state=2599520,\n",
       "                               reg_alpha=None, reg_lambda=None,\n",
       "                               scale_pos_weight=None,\n",
       "                               subsample=0.7109465438773837,\n",
       "                               tree_method='gpu_hist', ...))])"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Нужно увеличить n_estimators, так как параметры были найдены на уменьшенном наборе данных\n",
    "ml_pipe3.set_params(**par )# 'n_estimators' ml__learning_rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "supposed-practice",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:43.344951Z",
     "iopub.status.busy": "2021-04-25T10:51:43.336987Z",
     "iopub.status.idle": "2021-04-25T10:51:43.350567Z",
     "shell.execute_reply": "2021-04-25T10:51:43.350014Z"
    },
    "papermill": {
     "duration": 0.108292,
     "end_time": "2021-04-25T10:51:43.350700",
     "exception": false,
     "start_time": "2021-04-25T10:51:43.242408",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'memory': None,\n",
       " 'steps': [('tr',\n",
       "   ColumnTransformer(transformers=[('num', Pipeline(steps=[('scal', None)]),\n",
       "                                    ['Pclass', 'Age', 'SibSp', 'Parch', 'Fare']),\n",
       "                                   ('cat',\n",
       "                                    Pipeline(steps=[('enc', TargetEncoder())]),\n",
       "                                    Index(['Sex', 'Ticket', 'Cabin', 'Embarked'], dtype='object'))])),\n",
       "  ('ml',\n",
       "   XGBClassifier(alpha=0.32470257316374473, base_score=None, booster=None,\n",
       "                 colsample_bylevel=None, colsample_bynode=None,\n",
       "                 colsample_bytree=0.6248628797707747, device='gpu',\n",
       "                 gamma=0.9123913042030198, gpu_device_id=0, gpu_id=None,\n",
       "                 gpu_platform_id=0, importance_type='gain',\n",
       "                 interaction_constraints=None, lambda=0.6290623246094361,\n",
       "                 learning_rate=0.0005, max_delta_step=None, max_depth=6,\n",
       "                 min_child_weight=24, missing=nan, monotone_constraints=None,\n",
       "                 n_estimators=24000, n_jobs=None, num_parallel_tree=None,\n",
       "                 predictor='gpu_predictor', random_state=2599520, reg_alpha=None,\n",
       "                 reg_lambda=None, scale_pos_weight=None,\n",
       "                 subsample=0.7109465438773837, tree_method='gpu_hist', ...))],\n",
       " 'verbose': False,\n",
       " 'tr': ColumnTransformer(transformers=[('num', Pipeline(steps=[('scal', None)]),\n",
       "                                  ['Pclass', 'Age', 'SibSp', 'Parch', 'Fare']),\n",
       "                                 ('cat',\n",
       "                                  Pipeline(steps=[('enc', TargetEncoder())]),\n",
       "                                  Index(['Sex', 'Ticket', 'Cabin', 'Embarked'], dtype='object'))]),\n",
       " 'ml': XGBClassifier(alpha=0.32470257316374473, base_score=None, booster=None,\n",
       "               colsample_bylevel=None, colsample_bynode=None,\n",
       "               colsample_bytree=0.6248628797707747, device='gpu',\n",
       "               gamma=0.9123913042030198, gpu_device_id=0, gpu_id=None,\n",
       "               gpu_platform_id=0, importance_type='gain',\n",
       "               interaction_constraints=None, lambda=0.6290623246094361,\n",
       "               learning_rate=0.0005, max_delta_step=None, max_depth=6,\n",
       "               min_child_weight=24, missing=nan, monotone_constraints=None,\n",
       "               n_estimators=24000, n_jobs=None, num_parallel_tree=None,\n",
       "               predictor='gpu_predictor', random_state=2599520, reg_alpha=None,\n",
       "               reg_lambda=None, scale_pos_weight=None,\n",
       "               subsample=0.7109465438773837, tree_method='gpu_hist', ...),\n",
       " 'tr__n_jobs': None,\n",
       " 'tr__remainder': 'drop',\n",
       " 'tr__sparse_threshold': 0.3,\n",
       " 'tr__transformer_weights': None,\n",
       " 'tr__transformers': [('num',\n",
       "   Pipeline(steps=[('scal', None)]),\n",
       "   ['Pclass', 'Age', 'SibSp', 'Parch', 'Fare']),\n",
       "  ('cat',\n",
       "   Pipeline(steps=[('enc', TargetEncoder())]),\n",
       "   Index(['Sex', 'Ticket', 'Cabin', 'Embarked'], dtype='object'))],\n",
       " 'tr__verbose': False,\n",
       " 'tr__num': Pipeline(steps=[('scal', None)]),\n",
       " 'tr__cat': Pipeline(steps=[('enc', TargetEncoder())]),\n",
       " 'tr__num__memory': None,\n",
       " 'tr__num__steps': [('scal', None)],\n",
       " 'tr__num__verbose': False,\n",
       " 'tr__num__scal': None,\n",
       " 'tr__cat__memory': None,\n",
       " 'tr__cat__steps': [('enc', TargetEncoder())],\n",
       " 'tr__cat__verbose': False,\n",
       " 'tr__cat__enc': TargetEncoder(),\n",
       " 'tr__cat__enc__cols': None,\n",
       " 'tr__cat__enc__drop_invariant': False,\n",
       " 'tr__cat__enc__handle_missing': 'value',\n",
       " 'tr__cat__enc__handle_unknown': 'value',\n",
       " 'tr__cat__enc__min_samples_leaf': 1,\n",
       " 'tr__cat__enc__return_df': True,\n",
       " 'tr__cat__enc__smoothing': 1.0,\n",
       " 'tr__cat__enc__verbose': 0,\n",
       " 'ml__objective': 'binary:logistic',\n",
       " 'ml__use_label_encoder': True,\n",
       " 'ml__base_score': None,\n",
       " 'ml__booster': None,\n",
       " 'ml__colsample_bylevel': None,\n",
       " 'ml__colsample_bynode': None,\n",
       " 'ml__colsample_bytree': 0.6248628797707747,\n",
       " 'ml__gamma': 0.9123913042030198,\n",
       " 'ml__gpu_id': None,\n",
       " 'ml__importance_type': 'gain',\n",
       " 'ml__interaction_constraints': None,\n",
       " 'ml__learning_rate': 0.0005,\n",
       " 'ml__max_delta_step': None,\n",
       " 'ml__max_depth': 6,\n",
       " 'ml__min_child_weight': 24,\n",
       " 'ml__missing': nan,\n",
       " 'ml__monotone_constraints': None,\n",
       " 'ml__n_estimators': 24000,\n",
       " 'ml__n_jobs': None,\n",
       " 'ml__num_parallel_tree': None,\n",
       " 'ml__random_state': 2599520,\n",
       " 'ml__reg_alpha': None,\n",
       " 'ml__reg_lambda': None,\n",
       " 'ml__scale_pos_weight': None,\n",
       " 'ml__subsample': 0.7109465438773837,\n",
       " 'ml__tree_method': 'gpu_hist',\n",
       " 'ml__validate_parameters': None,\n",
       " 'ml__verbosity': None,\n",
       " 'ml__lambda': 0.6290623246094361,\n",
       " 'ml__alpha': 0.32470257316374473,\n",
       " 'ml__device': 'gpu',\n",
       " 'ml__gpu_platform_id': 0,\n",
       " 'ml__gpu_device_id': 0,\n",
       " 'ml__predictor': 'gpu_predictor'}"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ml_pipe3.get_params()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "published-diving",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:43.528343Z",
     "iopub.status.busy": "2021-04-25T10:51:43.519443Z",
     "iopub.status.idle": "2021-04-25T10:51:43.531989Z",
     "shell.execute_reply": "2021-04-25T10:51:43.531478Z"
    },
    "papermill": {
     "duration": 0.101971,
     "end_time": "2021-04-25T10:51:43.532161",
     "exception": false,
     "start_time": "2021-04-25T10:51:43.430190",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "with open(NAME_RUN_LOG, 'at', encoding='utf-8') as logfile:\n",
    "        print('Параметры конечной модели, формирующей посылку(submission): \\n{}'.format(ml_pipe3.get_params()), file=logfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "comfortable-shakespeare",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:51:43.724036Z",
     "iopub.status.busy": "2021-04-25T10:51:43.722853Z",
     "iopub.status.idle": "2021-04-25T10:52:54.365689Z",
     "shell.execute_reply": "2021-04-25T10:52:54.364505Z"
    },
    "id": "L_w96b7Sj56n",
    "papermill": {
     "duration": 70.74439,
     "end_time": "2021-04-25T10:52:54.365928",
     "exception": false,
     "start_time": "2021-04-25T10:51:43.621538",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/category_encoders/utils.py:21: FutureWarning: is_categorical is deprecated and will be removed in a future version.  Use is_categorical_dtype instead\n",
      "  elif pd.api.types.is_categorical(cols):\n",
      "/opt/conda/lib/python3.7/site-packages/xgboost/sklearn.py:888: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[10:51:44] WARNING: ../src/learner.cc:541: \n",
      "Parameters: { device, gpu_device_id, gpu_platform_id } might not be used.\n",
      "\n",
      "  This may not be accurate due to some parameters are only used in language bindings but\n",
      "  passed down to XGBoost core.  Or some parameters are not used but slip through this\n",
      "  verification. Please open an issue if you find above cases.\n",
      "\n",
      "\n",
      "[10:51:44] WARNING: ../src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "CPU times: user 1min 7s, sys: 1.56 s, total: 1min 9s\n",
      "Wall time: 1min 10s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('tr',\n",
       "                 ColumnTransformer(transformers=[('num',\n",
       "                                                  Pipeline(steps=[('scal',\n",
       "                                                                   None)]),\n",
       "                                                  ['Pclass', 'Age', 'SibSp',\n",
       "                                                   'Parch', 'Fare']),\n",
       "                                                 ('cat',\n",
       "                                                  Pipeline(steps=[('enc',\n",
       "                                                                   TargetEncoder())]),\n",
       "                                                  Index(['Sex', 'Ticket', 'Cabin', 'Embarked'], dtype='object'))])),\n",
       "                ('ml',\n",
       "                 XGBClassifier(alpha=0.32470257316374473, base_score=0.5,\n",
       "                               booster='gbtree', colsample_bylevel=1,\n",
       "                               colsamp...\n",
       "                               lambda=0.6290623246094361, learning_rate=0.0005,\n",
       "                               max_delta_step=0, max_depth=6,\n",
       "                               min_child_weight=24, missing=nan,\n",
       "                               monotone_constraints='()', n_estimators=24000,\n",
       "                               n_jobs=2, num_parallel_tree=1,\n",
       "                               predictor='gpu_predictor', random_state=2599520,\n",
       "                               reg_alpha=0.324702561, reg_lambda=0.629062295,\n",
       "                               scale_pos_weight=1, subsample=0.7109465438773837,\n",
       "                               tree_method='gpu_hist', ...))])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# обучаем обновленную модель на всей обучающей выборке\n",
    "ml_pipe3.fit(fulldata, y_fulldata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "earned-statement",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T10:52:54.547688Z",
     "iopub.status.busy": "2021-04-25T10:52:54.546670Z",
     "iopub.status.idle": "2021-04-25T11:10:28.931833Z",
     "shell.execute_reply": "2021-04-25T11:10:28.930859Z"
    },
    "papermill": {
     "duration": 1054.481417,
     "end_time": "2021-04-25T11:10:28.932051",
     "exception": false,
     "start_time": "2021-04-25T10:52:54.450634",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC на всей исторической выборке: 0.854\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([0.4423113465309143], [0.4423113465309143])"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# получаем ответы (вероятности) для проверочных блоков перекрестной проверки\n",
    "XGBM_probs = cross_val_predict(ml_pipe3, fulldata, y_fulldata, cv=10, n_jobs=-1, method='predict_proba')[:,1]\n",
    "# смотрим значение AUC\n",
    "print('AUC на всей исторической выборке: {:.3f}'.format(\n",
    "    roc_auc_score(y_fulldata, XGBM_probs )))\n",
    "\n",
    "# и выбираем лучшую точку порога\n",
    "def Find_Optimal_Cutoff(target, predicted):\n",
    "    fpr, tpr, threshold = roc_curve(target, predicted)\n",
    "    i = np.arange(len(tpr)) \n",
    "    roc = pd.DataFrame({'tf' : pd.Series(tpr-(1-fpr), index=i), 'threshold' : pd.Series(threshold, index=i)})\n",
    "    roc_t = roc.loc[(roc.tf-0).abs().argsort()[:1]]\n",
    "\n",
    "    return list(roc_t['threshold']) \n",
    "\n",
    "\n",
    "def Find_Optimal_Cutoff_2(target, predicted):\n",
    "    \"\"\" Find the optimal probability cutoff point for a classification model related to event rate\n",
    "    Parameters\n",
    "    ----------\n",
    "    target : Matrix with dependent or target data, where rows are observations\n",
    "\n",
    "    predicted : Matrix with predicted data, where rows are observations\n",
    "\n",
    "    Returns\n",
    "    -------     \n",
    "    list type, with optimal cutoff value\n",
    "        \n",
    "    \"\"\"\n",
    "    fpr, tpr, threshold = roc_curve(target, predicted)\n",
    "    i = np.arange(len(tpr)) \n",
    "    roc = pd.DataFrame({'tf' : pd.Series(tpr-(1-fpr), index=i), 'threshold' : pd.Series(threshold, index=i)})\n",
    "    roc_t = roc.iloc[(roc.tf-0).abs().argsort()[:1]]\n",
    "\n",
    "    return list(roc_t['threshold']) \n",
    "\n",
    "threshold = Find_Optimal_Cutoff(y_fulldata,XGBM_probs)\n",
    "threshold2 = Find_Optimal_Cutoff_2(y_fulldata,XGBM_probs)\n",
    "threshold, threshold2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "derived-harassment",
   "metadata": {
    "papermill": {
     "duration": 0.079961,
     "end_time": "2021-04-25T11:10:29.095853",
     "exception": false,
     "start_time": "2021-04-25T11:10:29.015892",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "permanent-initial",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T11:10:29.264518Z",
     "iopub.status.busy": "2021-04-25T11:10:29.263570Z",
     "iopub.status.idle": "2021-04-25T11:10:29.267499Z",
     "shell.execute_reply": "2021-04-25T11:10:29.266979Z"
    },
    "papermill": {
     "duration": 0.090244,
     "end_time": "2021-04-25T11:10:29.267646",
     "exception": false,
     "start_time": "2021-04-25T11:10:29.177402",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    " threshold = 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "under-nancy",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T11:10:29.443210Z",
     "iopub.status.busy": "2021-04-25T11:10:29.442065Z",
     "iopub.status.idle": "2021-04-25T11:10:30.223654Z",
     "shell.execute_reply": "2021-04-25T11:10:30.224283Z"
    },
    "id": "TvrX7Zx7j56n",
    "papermill": {
     "duration": 0.875383,
     "end_time": "2021-04-25T11:10:30.224551",
     "exception": false,
     "start_time": "2021-04-25T11:10:29.349168",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 780 ms, sys: 11.8 ms, total: 792 ms\n",
      "Wall time: 780 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# вычисляем вероятности для тестовой выборки\n",
    "ml_preds_prob = ml_pipe3.predict_proba(test_o)[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "starting-respondent",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T11:10:30.535420Z",
     "iopub.status.busy": "2021-04-25T11:10:30.533433Z",
     "iopub.status.idle": "2021-04-25T11:10:30.541477Z",
     "shell.execute_reply": "2021-04-25T11:10:30.540674Z"
    },
    "papermill": {
     "duration": 0.164362,
     "end_time": "2021-04-25T11:10:30.541744",
     "exception": false,
     "start_time": "2021-04-25T11:10:30.377382",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0, 1, 1, ..., 0, 1, 1]), numpy.ndarray)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_pred_bool = (ml_preds_prob >= threshold).astype(int)\n",
    "test_pred_bool,type(test_pred_bool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "republican-polish",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T11:10:30.850555Z",
     "iopub.status.busy": "2021-04-25T11:10:30.849052Z",
     "iopub.status.idle": "2021-04-25T11:10:31.438953Z",
     "shell.execute_reply": "2021-04-25T11:10:31.438216Z"
    },
    "id": "HNHBkKGvj56o",
    "papermill": {
     "duration": 0.748349,
     "end_time": "2021-04-25T11:10:31.439132",
     "exception": false,
     "start_time": "2021-04-25T11:10:30.690783",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Survived</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PassengerId</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>100000</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100001</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100002</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100003</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100004</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199995</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199996</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199997</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199998</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199999</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100000 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             Survived\n",
       "PassengerId          \n",
       "100000              0\n",
       "100001              1\n",
       "100002              1\n",
       "100003              0\n",
       "100004              1\n",
       "...               ...\n",
       "199995              1\n",
       "199996              0\n",
       "199997              0\n",
       "199998              1\n",
       "199999              1\n",
       "\n",
       "[100000 rows x 1 columns]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission = pd.read_csv(input_path / ('sample_submission.' + d_ext), index_col=INDEX_COL)\n",
    "####### и формируем посылку\n",
    "submission[TARGET] = test_pred_bool\n",
    "if  WHERE_PLACE == \"kaggle\":\n",
    "    submission.to_csv(NAME_OUTPUT_FILE+'.'+d_ext)\n",
    "else:    \n",
    "    submission.to_csv(input_path / (NAME_OUTPUT_FILE+'.'+d_ext))\n",
    "\n",
    "submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "desirable-colombia",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T11:10:31.631687Z",
     "iopub.status.busy": "2021-04-25T11:10:31.630516Z",
     "iopub.status.idle": "2021-04-25T11:10:31.637099Z",
     "shell.execute_reply": "2021-04-25T11:10:31.637952Z"
    },
    "id": "qnHI4879j56o",
    "papermill": {
     "duration": 0.108114,
     "end_time": "2021-04-25T11:10:31.638168",
     "exception": false,
     "start_time": "2021-04-25T11:10:31.530054",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Duration: 0.0 hours 19.0 minutes 22.73771071434021 seconds.\n",
      "\n",
      "\n",
      "Время окончания работы скрипта: 25-04-2021 11:10\n",
      "***********************\n",
      "\n"
     ]
    }
   ],
   "source": [
    "end_time = time()\n",
    "time_taken = end_time - start_time # time_taken is in seconds\n",
    "hours, rest = divmod(time_taken,3600)\n",
    "minutes, seconds = divmod(rest, 60)\n",
    "print('Duration: {} hours {} minutes {} seconds.'.format(hours, minutes, seconds))\n",
    "print('\\n\\nВремя окончания работы скрипта: {}\\n***********************\\n'.format(datetime.datetime.now().strftime(\"%d-%m-%Y %H:%M\")))\n",
    "with open(NAME_RUN_LOG, 'at', encoding='utf-8') as logfile:\n",
    "    print('\\n\\nВремя окончания работы скрипта: {}\\n***********************\\n'.format(datetime.datetime.now().strftime(\"%d-%m-%Y %H:%M\")), \n",
    "          file=logfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "coordinate-proof",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T11:10:31.836825Z",
     "iopub.status.busy": "2021-04-25T11:10:31.835742Z",
     "iopub.status.idle": "2021-04-25T11:10:31.861656Z",
     "shell.execute_reply": "2021-04-25T11:10:31.862301Z"
    },
    "papermill": {
     "duration": 0.127657,
     "end_time": "2021-04-25T11:10:31.862476",
     "exception": false,
     "start_time": "2021-04-25T11:10:31.734819",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'memory': None,\n",
       " 'steps': [('tr',\n",
       "   ColumnTransformer(transformers=[('num', Pipeline(steps=[('scal', None)]),\n",
       "                                    ['Pclass', 'Age', 'SibSp', 'Parch', 'Fare']),\n",
       "                                   ('cat',\n",
       "                                    Pipeline(steps=[('enc', TargetEncoder())]),\n",
       "                                    Index(['Sex', 'Ticket', 'Cabin', 'Embarked'], dtype='object'))])),\n",
       "  ('ml',\n",
       "   XGBClassifier(alpha=0.32470257316374473, base_score=0.5, booster='gbtree',\n",
       "                 colsample_bylevel=1, colsample_bynode=1,\n",
       "                 colsample_bytree=0.6248628797707747, device='gpu',\n",
       "                 gamma=0.9123913042030198, gpu_device_id=0, gpu_id=0,\n",
       "                 gpu_platform_id=0, importance_type='gain',\n",
       "                 interaction_constraints='', lambda=0.6290623246094361,\n",
       "                 learning_rate=0.0005, max_delta_step=0, max_depth=6,\n",
       "                 min_child_weight=24, missing=nan, monotone_constraints='()',\n",
       "                 n_estimators=24000, n_jobs=2, num_parallel_tree=1,\n",
       "                 predictor='gpu_predictor', random_state=2599520,\n",
       "                 reg_alpha=0.324702561, reg_lambda=0.629062295, scale_pos_weight=1,\n",
       "                 subsample=0.7109465438773837, tree_method='gpu_hist', ...))],\n",
       " 'verbose': False,\n",
       " 'tr': ColumnTransformer(transformers=[('num', Pipeline(steps=[('scal', None)]),\n",
       "                                  ['Pclass', 'Age', 'SibSp', 'Parch', 'Fare']),\n",
       "                                 ('cat',\n",
       "                                  Pipeline(steps=[('enc', TargetEncoder())]),\n",
       "                                  Index(['Sex', 'Ticket', 'Cabin', 'Embarked'], dtype='object'))]),\n",
       " 'ml': XGBClassifier(alpha=0.32470257316374473, base_score=0.5, booster='gbtree',\n",
       "               colsample_bylevel=1, colsample_bynode=1,\n",
       "               colsample_bytree=0.6248628797707747, device='gpu',\n",
       "               gamma=0.9123913042030198, gpu_device_id=0, gpu_id=0,\n",
       "               gpu_platform_id=0, importance_type='gain',\n",
       "               interaction_constraints='', lambda=0.6290623246094361,\n",
       "               learning_rate=0.0005, max_delta_step=0, max_depth=6,\n",
       "               min_child_weight=24, missing=nan, monotone_constraints='()',\n",
       "               n_estimators=24000, n_jobs=2, num_parallel_tree=1,\n",
       "               predictor='gpu_predictor', random_state=2599520,\n",
       "               reg_alpha=0.324702561, reg_lambda=0.629062295, scale_pos_weight=1,\n",
       "               subsample=0.7109465438773837, tree_method='gpu_hist', ...),\n",
       " 'tr__n_jobs': None,\n",
       " 'tr__remainder': 'drop',\n",
       " 'tr__sparse_threshold': 0.3,\n",
       " 'tr__transformer_weights': None,\n",
       " 'tr__transformers': [('num',\n",
       "   Pipeline(steps=[('scal', None)]),\n",
       "   ['Pclass', 'Age', 'SibSp', 'Parch', 'Fare']),\n",
       "  ('cat',\n",
       "   Pipeline(steps=[('enc', TargetEncoder())]),\n",
       "   Index(['Sex', 'Ticket', 'Cabin', 'Embarked'], dtype='object'))],\n",
       " 'tr__verbose': False,\n",
       " 'tr__num': Pipeline(steps=[('scal', None)]),\n",
       " 'tr__cat': Pipeline(steps=[('enc', TargetEncoder())]),\n",
       " 'tr__num__memory': None,\n",
       " 'tr__num__steps': [('scal', None)],\n",
       " 'tr__num__verbose': False,\n",
       " 'tr__num__scal': None,\n",
       " 'tr__cat__memory': None,\n",
       " 'tr__cat__steps': [('enc', TargetEncoder())],\n",
       " 'tr__cat__verbose': False,\n",
       " 'tr__cat__enc': TargetEncoder(),\n",
       " 'tr__cat__enc__cols': None,\n",
       " 'tr__cat__enc__drop_invariant': False,\n",
       " 'tr__cat__enc__handle_missing': 'value',\n",
       " 'tr__cat__enc__handle_unknown': 'value',\n",
       " 'tr__cat__enc__min_samples_leaf': 1,\n",
       " 'tr__cat__enc__return_df': True,\n",
       " 'tr__cat__enc__smoothing': 1.0,\n",
       " 'tr__cat__enc__verbose': 0,\n",
       " 'ml__objective': 'binary:logistic',\n",
       " 'ml__use_label_encoder': True,\n",
       " 'ml__base_score': 0.5,\n",
       " 'ml__booster': 'gbtree',\n",
       " 'ml__colsample_bylevel': 1,\n",
       " 'ml__colsample_bynode': 1,\n",
       " 'ml__colsample_bytree': 0.6248628797707747,\n",
       " 'ml__gamma': 0.9123913042030198,\n",
       " 'ml__gpu_id': 0,\n",
       " 'ml__importance_type': 'gain',\n",
       " 'ml__interaction_constraints': '',\n",
       " 'ml__learning_rate': 0.0005,\n",
       " 'ml__max_delta_step': 0,\n",
       " 'ml__max_depth': 6,\n",
       " 'ml__min_child_weight': 24,\n",
       " 'ml__missing': nan,\n",
       " 'ml__monotone_constraints': '()',\n",
       " 'ml__n_estimators': 24000,\n",
       " 'ml__n_jobs': 2,\n",
       " 'ml__num_parallel_tree': 1,\n",
       " 'ml__random_state': 2599520,\n",
       " 'ml__reg_alpha': 0.324702561,\n",
       " 'ml__reg_lambda': 0.629062295,\n",
       " 'ml__scale_pos_weight': 1,\n",
       " 'ml__subsample': 0.7109465438773837,\n",
       " 'ml__tree_method': 'gpu_hist',\n",
       " 'ml__validate_parameters': 1,\n",
       " 'ml__verbosity': None,\n",
       " 'ml__lambda': 0.6290623246094361,\n",
       " 'ml__alpha': 0.32470257316374473,\n",
       " 'ml__device': 'gpu',\n",
       " 'ml__gpu_platform_id': 0,\n",
       " 'ml__gpu_device_id': 0,\n",
       " 'ml__predictor': 'gpu_predictor'}"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ml_pipe3.get_params()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "leading-solid",
   "metadata": {
    "papermill": {
     "duration": 0.092869,
     "end_time": "2021-04-25T11:10:32.052444",
     "exception": false,
     "start_time": "2021-04-25T11:10:31.959575",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# ### Adjusting over the entire historical sample Final phase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "alive-hamilton",
   "metadata": {
    "papermill": {
     "duration": 0.093448,
     "end_time": "2021-04-25T11:10:32.236844",
     "exception": false,
     "start_time": "2021-04-25T11:10:32.143396",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "architectural-edwards",
   "metadata": {
    "papermill": {
     "duration": 0.086982,
     "end_time": "2021-04-25T11:10:32.409572",
     "exception": false,
     "start_time": "2021-04-25T11:10:32.322590",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Для использование в дальнейшем стекинге:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "competent-elder",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T11:10:32.588502Z",
     "iopub.status.busy": "2021-04-25T11:10:32.587352Z",
     "iopub.status.idle": "2021-04-25T11:10:33.431798Z",
     "shell.execute_reply": "2021-04-25T11:10:33.432396Z"
    },
    "papermill": {
     "duration": 0.937907,
     "end_time": "2021-04-25T11:10:33.432590",
     "exception": false,
     "start_time": "2021-04-25T11:10:32.494683",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.09754119, 0.53959179, 0.92725462, 0.2233218 , 0.78792638,\n",
       "       0.23304656, 0.74515748, 0.1275479 , 0.40948924, 0.12997659])"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.savetxt(NAME_OUTPUT_FOR_STAKING_TEST, ml_preds_prob )\n",
    "# Проверим\n",
    "new_num_arr = np.loadtxt(NAME_OUTPUT_FOR_STAKING_TEST)\n",
    "new_num_arr[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "alike-advertiser",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-04-25T11:10:33.641287Z",
     "iopub.status.busy": "2021-04-25T11:10:33.640214Z",
     "iopub.status.idle": "2021-04-25T11:10:34.470701Z",
     "shell.execute_reply": "2021-04-25T11:10:34.471217Z"
    },
    "papermill": {
     "duration": 0.920739,
     "end_time": "2021-04-25T11:10:34.471551",
     "exception": false,
     "start_time": "2021-04-25T11:10:33.550812",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.29751891, 0.08065243, 0.11827352, 0.10194964, 0.09276646,\n",
       "       0.12814917, 0.10112795, 0.46475929, 0.23735365, 0.66035628])"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#XGBM_probs.to_csv(\"XGBM_probs_for_staking(v1)_TEST\"+'.'+d_ext)\n",
    "np.savetxt(NAME_OUTPUT_FOR_STAKING, XGBM_probs )\n",
    "# Проверим\n",
    "new_num_arr2 = np.loadtxt(NAME_OUTPUT_FOR_STAKING)\n",
    "new_num_arr2[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "explicit-rating",
   "metadata": {
    "papermill": {
     "duration": 0.085876,
     "end_time": "2021-04-25T11:10:34.645302",
     "exception": false,
     "start_time": "2021-04-25T11:10:34.559426",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 1175.250436,
   "end_time": "2021-04-25T11:10:37.128577",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-04-25T10:51:01.878141",
   "version": "2.3.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
